MCPcopy Create free account
hub / github.com/apache/arrow / test_dataset_partition_with_slash

Function test_dataset_partition_with_slash

python/pyarrow/tests/test_dataset.py:5693–5727  ·  view source on GitHub ↗
(tmpdir)

Source from the content-addressed store, hash-verified

5691
5692
5693def test_dataset_partition_with_slash(tmpdir):
5694 from pyarrow import dataset as ds
5695
5696 path = tmpdir / "slash-writer-x"
5697
5698 dt_table = pa.Table.from_arrays([
5699 pa.array([1, 2, 3, 4, 5], pa.int32()),
5700 pa.array(["experiment/A/f.csv", "experiment/B/f.csv",
5701 "experiment/A/f.csv", "experiment/C/k.csv",
5702 "experiment/M/i.csv"], pa.utf8())], ["exp_id", "exp_meta"])
5703
5704 ds.write_dataset(
5705 data=dt_table,
5706 base_dir=path,
5707 format='ipc',
5708 partitioning=['exp_meta'],
5709 partitioning_flavor='hive',
5710 )
5711
5712 read_table = ds.dataset(
5713 source=path,
5714 format='ipc',
5715 partitioning='hive',
5716 schema=pa.schema([pa.field("exp_id", pa.int32()),
5717 pa.field("exp_meta", pa.utf8())])
5718 ).to_table().combine_chunks()
5719
5720 assert dt_table == read_table.sort_by("exp_id")
5721
5722 exp_meta = dt_table.column(1).to_pylist()
5723 exp_meta = sorted(set(exp_meta)) # take unique
5724 encoded_paths = ["exp_meta=" + quote(path, safe='') for path in exp_meta]
5725 file_paths = sorted(os.listdir(path))
5726
5727 assert encoded_paths == file_paths
5728
5729
5730@pytest.mark.parquet

Callers

nothing calls this directly

Calls 5

arrayMethod · 0.45
to_tableMethod · 0.45
schemaMethod · 0.45
fieldMethod · 0.45
columnMethod · 0.45

Tested by

no test coverage detected