MCPcopy Create free account
hub / github.com/apache/arrow / test_write_dataset_existing_data

Function test_write_dataset_existing_data

python/pyarrow/tests/test_dataset.py:4441–4486  ·  view source on GitHub ↗
(tempdir)

Source from the content-addressed store, hash-verified

4439
4440@pytest.mark.pandas
4441def test_write_dataset_existing_data(tempdir):
4442 directory = tempdir / 'ds'
4443 table = pa.table({'b': ['x', 'y', 'z'], 'c': [1, 2, 3]})
4444 partitioning = ds.partitioning(schema=pa.schema(
4445 [pa.field('c', pa.int64())]), flavor='hive')
4446
4447 def compare_tables_ignoring_order(t1, t2):
4448 df1 = t1.to_pandas().sort_values('b').reset_index(drop=True)
4449 df2 = t2.to_pandas().sort_values('b').reset_index(drop=True)
4450 assert df1.equals(df2)
4451
4452 # First write is ok
4453 ds.write_dataset(table, directory, partitioning=partitioning, format='ipc')
4454
4455 table = pa.table({'b': ['a', 'b', 'c'], 'c': [2, 3, 4]})
4456
4457 # Second write should fail
4458 with pytest.raises(pa.ArrowInvalid):
4459 ds.write_dataset(table, directory,
4460 partitioning=partitioning, format='ipc')
4461
4462 extra_table = pa.table({'b': ['e']})
4463 extra_file = directory / 'c=2' / 'foo.arrow'
4464 pyarrow.feather.write_feather(extra_table, extra_file)
4465
4466 # Should be ok and overwrite with overwrite behavior
4467 ds.write_dataset(table, directory, partitioning=partitioning,
4468 format='ipc',
4469 existing_data_behavior='overwrite_or_ignore')
4470
4471 overwritten = pa.table(
4472 {'b': ['e', 'x', 'a', 'b', 'c'], 'c': [2, 1, 2, 3, 4]})
4473 readback = ds.dataset(tempdir, format='ipc',
4474 partitioning=partitioning).to_table()
4475 compare_tables_ignoring_order(readback, overwritten)
4476 assert extra_file.exists()
4477
4478 # Should be ok and delete matching with delete_matching
4479 ds.write_dataset(table, directory, partitioning=partitioning,
4480 format='ipc', existing_data_behavior='delete_matching')
4481
4482 overwritten = pa.table({'b': ['x', 'a', 'b', 'c'], 'c': [1, 2, 3, 4]})
4483 readback = ds.dataset(tempdir, format='ipc',
4484 partitioning=partitioning).to_table()
4485 compare_tables_ignoring_order(readback, overwritten)
4486 assert not extra_file.exists()
4487
4488
4489def _generate_random_int_array(size=4, min=1, max=10):

Callers

nothing calls this directly

Calls 5

partitioningMethod · 0.80
schemaMethod · 0.45
fieldMethod · 0.45
to_tableMethod · 0.45

Tested by

no test coverage detected