(tempdir)
| 4110 | @pytest.mark.pandas |
| 4111 | @pytest.mark.parquet |
| 4112 | def test_dataset_preserved_partitioning(tempdir): |
| 4113 | # ARROW-8655 |
| 4114 | |
| 4115 | # through discovery, but without partitioning |
| 4116 | _, path = _create_single_file(tempdir) |
| 4117 | dataset = ds.dataset(path) |
| 4118 | assert isinstance(dataset.partitioning, ds.DirectoryPartitioning) |
| 4119 | # TODO(GH-34884) partitioning attribute not preserved in pickling |
| 4120 | # dataset_ = ds.dataset(path) |
| 4121 | # for dataset in [dataset_, pickle_module.loads(pickle_module.dumps(dataset_))]: |
| 4122 | # assert isinstance(dataset.partitioning, ds.DirectoryPartitioning) |
| 4123 | |
| 4124 | # through discovery, with hive partitioning but not specified |
| 4125 | full_table, path = _create_partitioned_dataset(tempdir) |
| 4126 | dataset = ds.dataset(path) |
| 4127 | assert isinstance(dataset.partitioning, ds.DirectoryPartitioning) |
| 4128 | |
| 4129 | # through discovery, with hive partitioning (from a partitioning factory) |
| 4130 | dataset = ds.dataset(path, partitioning="hive") |
| 4131 | part = dataset.partitioning |
| 4132 | assert part is not None |
| 4133 | assert isinstance(part, ds.HivePartitioning) |
| 4134 | assert part.schema == pa.schema([("part", pa.int32())]) |
| 4135 | assert len(part.dictionaries) == 1 |
| 4136 | assert part.dictionaries[0] == pa.array([0, 1, 2], pa.int32()) |
| 4137 | |
| 4138 | # through discovery, with hive partitioning (from a partitioning object) |
| 4139 | part = ds.partitioning(pa.schema([("part", pa.int32())]), flavor="hive") |
| 4140 | assert isinstance(part, ds.HivePartitioning) # not a factory |
| 4141 | assert len(part.dictionaries) == 1 |
| 4142 | assert all(x is None for x in part.dictionaries) |
| 4143 | dataset = ds.dataset(path, partitioning=part) |
| 4144 | part = dataset.partitioning |
| 4145 | assert isinstance(part, ds.HivePartitioning) |
| 4146 | assert part.schema == pa.schema([("part", pa.int32())]) |
| 4147 | # TODO is this expected? |
| 4148 | assert len(part.dictionaries) == 1 |
| 4149 | assert all(x is None for x in part.dictionaries) |
| 4150 | |
| 4151 | # through manual creation -> not available |
| 4152 | dataset = ds.dataset(path, partitioning="hive") |
| 4153 | dataset2 = ds.FileSystemDataset( |
| 4154 | list(dataset.get_fragments()), schema=dataset.schema, |
| 4155 | format=dataset.format, filesystem=dataset.filesystem |
| 4156 | ) |
| 4157 | assert dataset2.partitioning is None |
| 4158 | |
| 4159 | # through discovery with ParquetDatasetFactory |
| 4160 | root_path = tempdir / "data-partitioned-metadata" |
| 4161 | metadata_path, _ = _create_parquet_dataset_partitioned(root_path) |
| 4162 | dataset = ds.parquet_dataset(metadata_path, partitioning="hive") |
| 4163 | part = dataset.partitioning |
| 4164 | assert part is not None |
| 4165 | assert isinstance(part, ds.HivePartitioning) |
| 4166 | assert part.schema == pa.schema([("part", pa.string())]) |
| 4167 | assert len(part.dictionaries) == 1 |
| 4168 | # will be fixed by ARROW-13153 (order is not preserved at the moment) |
| 4169 | # assert part.dictionaries[0] == pa.array(["a", "b"], pa.string()) |
nothing calls this directly
no test coverage detected