(tempdir, dataset_reader)
| 3563 | |
| 3564 | |
| 3565 | def test_csv_fragment_options(tempdir, dataset_reader): |
| 3566 | path = str(tempdir / 'test.csv') |
| 3567 | with open(path, 'w') as sink: |
| 3568 | sink.write('col0\nfoo\nspam\nMYNULL\n') |
| 3569 | dataset = ds.dataset(path, format='csv') |
| 3570 | convert_options = pyarrow.csv.ConvertOptions(null_values=['MYNULL'], |
| 3571 | strings_can_be_null=True) |
| 3572 | options = ds.CsvFragmentScanOptions( |
| 3573 | convert_options=convert_options, |
| 3574 | read_options=pa.csv.ReadOptions(block_size=2**16)) |
| 3575 | result = dataset_reader.to_table(dataset, fragment_scan_options=options) |
| 3576 | assert result.equals(pa.table({'col0': pa.array(['foo', 'spam', None])})) |
| 3577 | |
| 3578 | csv_format = ds.CsvFileFormat(convert_options=convert_options) |
| 3579 | dataset = ds.dataset(path, format=csv_format) |
| 3580 | result = dataset_reader.to_table(dataset) |
| 3581 | assert result.equals(pa.table({'col0': pa.array(['foo', 'spam', None])})) |
| 3582 | |
| 3583 | options = ds.CsvFragmentScanOptions() |
| 3584 | result = dataset_reader.to_table(dataset, fragment_scan_options=options) |
| 3585 | assert result.equals( |
| 3586 | pa.table({'col0': pa.array(['foo', 'spam', 'MYNULL'])})) |
| 3587 | |
| 3588 | |
| 3589 | @pytest.mark.pandas |
nothing calls this directly
no test coverage detected