| 3657 | |
| 3658 | |
| 3659 | def test_encoding(tempdir, dataset_reader): |
| 3660 | path = str(tempdir / 'test.csv') |
| 3661 | |
| 3662 | for encoding, input_rows in [ |
| 3663 | ('latin-1', b"a,b\nun,\xe9l\xe9phant"), |
| 3664 | ('utf16', b'\xff\xfea\x00,\x00b\x00\n\x00u\x00n\x00,' |
| 3665 | b'\x00\xe9\x00l\x00\xe9\x00p\x00h\x00a\x00n\x00t\x00'), |
| 3666 | ]: |
| 3667 | |
| 3668 | with open(path, 'wb') as sink: |
| 3669 | sink.write(input_rows) |
| 3670 | |
| 3671 | # Interpret as utf8: |
| 3672 | expected_schema = pa.schema([("a", pa.string()), ("b", pa.string())]) |
| 3673 | expected_table = pa.table({'a': ["un"], |
| 3674 | 'b': ["éléphant"]}, schema=expected_schema) |
| 3675 | |
| 3676 | read_options = pa.csv.ReadOptions(encoding=encoding) |
| 3677 | file_format = ds.CsvFileFormat(read_options=read_options) |
| 3678 | dataset_transcoded = ds.dataset(path, format=file_format) |
| 3679 | assert dataset_transcoded.schema.equals(expected_schema) |
| 3680 | assert dataset_transcoded.to_table().equals(expected_table) |
| 3681 | |
| 3682 | |
| 3683 | # Test if a dataset with non-utf8 chars in the column names is properly handled |