| 5691 | |
| 5692 | |
| 5693 | def test_dataset_partition_with_slash(tmpdir): |
| 5694 | from pyarrow import dataset as ds |
| 5695 | |
| 5696 | path = tmpdir / "slash-writer-x" |
| 5697 | |
| 5698 | dt_table = pa.Table.from_arrays([ |
| 5699 | pa.array([1, 2, 3, 4, 5], pa.int32()), |
| 5700 | pa.array(["experiment/A/f.csv", "experiment/B/f.csv", |
| 5701 | "experiment/A/f.csv", "experiment/C/k.csv", |
| 5702 | "experiment/M/i.csv"], pa.utf8())], ["exp_id", "exp_meta"]) |
| 5703 | |
| 5704 | ds.write_dataset( |
| 5705 | data=dt_table, |
| 5706 | base_dir=path, |
| 5707 | format='ipc', |
| 5708 | partitioning=['exp_meta'], |
| 5709 | partitioning_flavor='hive', |
| 5710 | ) |
| 5711 | |
| 5712 | read_table = ds.dataset( |
| 5713 | source=path, |
| 5714 | format='ipc', |
| 5715 | partitioning='hive', |
| 5716 | schema=pa.schema([pa.field("exp_id", pa.int32()), |
| 5717 | pa.field("exp_meta", pa.utf8())]) |
| 5718 | ).to_table().combine_chunks() |
| 5719 | |
| 5720 | assert dt_table == read_table.sort_by("exp_id") |
| 5721 | |
| 5722 | exp_meta = dt_table.column(1).to_pylist() |
| 5723 | exp_meta = sorted(set(exp_meta)) # take unique |
| 5724 | encoded_paths = ["exp_meta=" + quote(path, safe='') for path in exp_meta] |
| 5725 | file_paths = sorted(os.listdir(path)) |
| 5726 | |
| 5727 | assert encoded_paths == file_paths |
| 5728 | |
| 5729 | |
| 5730 | @pytest.mark.parquet |