(tempdir)
| 4439 | |
| 4440 | @pytest.mark.pandas |
| 4441 | def test_write_dataset_existing_data(tempdir): |
| 4442 | directory = tempdir / 'ds' |
| 4443 | table = pa.table({'b': ['x', 'y', 'z'], 'c': [1, 2, 3]}) |
| 4444 | partitioning = ds.partitioning(schema=pa.schema( |
| 4445 | [pa.field('c', pa.int64())]), flavor='hive') |
| 4446 | |
| 4447 | def compare_tables_ignoring_order(t1, t2): |
| 4448 | df1 = t1.to_pandas().sort_values('b').reset_index(drop=True) |
| 4449 | df2 = t2.to_pandas().sort_values('b').reset_index(drop=True) |
| 4450 | assert df1.equals(df2) |
| 4451 | |
| 4452 | # First write is ok |
| 4453 | ds.write_dataset(table, directory, partitioning=partitioning, format='ipc') |
| 4454 | |
| 4455 | table = pa.table({'b': ['a', 'b', 'c'], 'c': [2, 3, 4]}) |
| 4456 | |
| 4457 | # Second write should fail |
| 4458 | with pytest.raises(pa.ArrowInvalid): |
| 4459 | ds.write_dataset(table, directory, |
| 4460 | partitioning=partitioning, format='ipc') |
| 4461 | |
| 4462 | extra_table = pa.table({'b': ['e']}) |
| 4463 | extra_file = directory / 'c=2' / 'foo.arrow' |
| 4464 | pyarrow.feather.write_feather(extra_table, extra_file) |
| 4465 | |
| 4466 | # Should be ok and overwrite with overwrite behavior |
| 4467 | ds.write_dataset(table, directory, partitioning=partitioning, |
| 4468 | format='ipc', |
| 4469 | existing_data_behavior='overwrite_or_ignore') |
| 4470 | |
| 4471 | overwritten = pa.table( |
| 4472 | {'b': ['e', 'x', 'a', 'b', 'c'], 'c': [2, 1, 2, 3, 4]}) |
| 4473 | readback = ds.dataset(tempdir, format='ipc', |
| 4474 | partitioning=partitioning).to_table() |
| 4475 | compare_tables_ignoring_order(readback, overwritten) |
| 4476 | assert extra_file.exists() |
| 4477 | |
| 4478 | # Should be ok and delete matching with delete_matching |
| 4479 | ds.write_dataset(table, directory, partitioning=partitioning, |
| 4480 | format='ipc', existing_data_behavior='delete_matching') |
| 4481 | |
| 4482 | overwritten = pa.table({'b': ['x', 'a', 'b', 'c'], 'c': [1, 2, 3, 4]}) |
| 4483 | readback = ds.dataset(tempdir, format='ipc', |
| 4484 | partitioning=partitioning).to_table() |
| 4485 | compare_tables_ignoring_order(readback, overwritten) |
| 4486 | assert not extra_file.exists() |
| 4487 | |
| 4488 | |
| 4489 | def _generate_random_int_array(size=4, min=1, max=10): |
nothing calls this directly
no test coverage detected