| 3651 | |
| 3652 | |
| 3653 | def test_encoding(tempdir, dataset_reader): |
| 3654 | path = str(tempdir / 'test.csv') |
| 3655 | |
| 3656 | for encoding, input_rows in [ |
| 3657 | ('latin-1', b"a,b\nun,\xe9l\xe9phant"), |
| 3658 | ('utf16', b'\xff\xfea\x00,\x00b\x00\n\x00u\x00n\x00,' |
| 3659 | b'\x00\xe9\x00l\x00\xe9\x00p\x00h\x00a\x00n\x00t\x00'), |
| 3660 | ]: |
| 3661 | |
| 3662 | with open(path, 'wb') as sink: |
| 3663 | sink.write(input_rows) |
| 3664 | |
| 3665 | # Interpret as utf8: |
| 3666 | expected_schema = pa.schema([("a", pa.string()), ("b", pa.string())]) |
| 3667 | expected_table = pa.table({'a': ["un"], |
| 3668 | 'b': ["éléphant"]}, schema=expected_schema) |
| 3669 | |
| 3670 | read_options = pa.csv.ReadOptions(encoding=encoding) |
| 3671 | file_format = ds.CsvFileFormat(read_options=read_options) |
| 3672 | dataset_transcoded = ds.dataset(path, format=file_format) |
| 3673 | assert dataset_transcoded.schema.equals(expected_schema) |
| 3674 | assert dataset_transcoded.to_table().equals(expected_table) |
| 3675 | |
| 3676 | |
| 3677 | # Test if a dataset with non-utf8 chars in the column names is properly handled |