| 3652 | |
| 3653 | |
| 3654 | def test_encoding(tempdir, dataset_reader): |
| 3655 | path = str(tempdir / 'test.csv') |
| 3656 | |
| 3657 | for encoding, input_rows in [ |
| 3658 | ('latin-1', b"a,b\nun,\xe9l\xe9phant"), |
| 3659 | ('utf16', b'\xff\xfea\x00,\x00b\x00\n\x00u\x00n\x00,' |
| 3660 | b'\x00\xe9\x00l\x00\xe9\x00p\x00h\x00a\x00n\x00t\x00'), |
| 3661 | ]: |
| 3662 | |
| 3663 | with open(path, 'wb') as sink: |
| 3664 | sink.write(input_rows) |
| 3665 | |
| 3666 | # Interpret as utf8: |
| 3667 | expected_schema = pa.schema([("a", pa.string()), ("b", pa.string())]) |
| 3668 | expected_table = pa.table({'a': ["un"], |
| 3669 | 'b': ["éléphant"]}, schema=expected_schema) |
| 3670 | |
| 3671 | read_options = pa.csv.ReadOptions(encoding=encoding) |
| 3672 | file_format = ds.CsvFileFormat(read_options=read_options) |
| 3673 | dataset_transcoded = ds.dataset(path, format=file_format) |
| 3674 | assert dataset_transcoded.schema.equals(expected_schema) |
| 3675 | assert dataset_transcoded.to_table().equals(expected_table) |
| 3676 | |
| 3677 | |
| 3678 | # Test if a dataset with non-utf8 chars in the column names is properly handled |