| 3677 | |
| 3678 | # Test if a dataset with non-utf8 chars in the column names is properly handled |
| 3679 | def test_column_names_encoding(tempdir, dataset_reader): |
| 3680 | path = str(tempdir / 'test.csv') |
| 3681 | |
| 3682 | with open(path, 'wb') as sink: |
| 3683 | sink.write(b"\xe9,b\nun,\xe9l\xe9phant") |
| 3684 | |
| 3685 | # Interpret as utf8: |
| 3686 | expected_schema = pa.schema([("é", pa.string()), ("b", pa.string())]) |
| 3687 | expected_table = pa.table({'é': ["un"], |
| 3688 | 'b': ["éléphant"]}, schema=expected_schema) |
| 3689 | |
| 3690 | # Reading as string without specifying encoding should produce an error |
| 3691 | dataset = ds.dataset(path, format='csv', schema=expected_schema) |
| 3692 | with pytest.raises(pyarrow.lib.ArrowInvalid, match="invalid UTF8"): |
| 3693 | dataset_reader.to_table(dataset) |
| 3694 | |
| 3695 | # Setting the encoding in the read_options should transcode the data |
| 3696 | read_options = pa.csv.ReadOptions(encoding='latin-1') |
| 3697 | file_format = ds.CsvFileFormat(read_options=read_options) |
| 3698 | dataset_transcoded = ds.dataset(path, format=file_format) |
| 3699 | assert dataset_transcoded.schema.equals(expected_schema) |
| 3700 | assert dataset_transcoded.to_table().equals(expected_table) |
| 3701 | |
| 3702 | |
| 3703 | @pytest.mark.filterwarnings("ignore:pyarrow.feather:FutureWarning") |