MCPcopy Create free account
hub / github.com/apache/arrow / test_partition_discovery

Function test_partition_discovery

python/pyarrow/tests/test_dataset.py:2842–2896  ·  view source on GitHub ↗
(
    tempdir, partitioning, null_fallback, infer_dictionary, partition_keys
)

Source from the content-addressed store, hash-verified

2840 ([None, 2, 3], [None, 2, 3]),
2841])
2842def test_partition_discovery(
2843 tempdir, partitioning, null_fallback, infer_dictionary, partition_keys
2844):
2845 # ARROW-9288 / ARROW-9476
2846 table = pa.table({'a': range(9), 'b': [0.0] * 4 + [1.0] * 5})
2847
2848 has_null = None in partition_keys[0] or None in partition_keys[1]
2849 if partitioning == "directory" and has_null:
2850 # Directory partitioning can't handle the first part being null
2851 return
2852
2853 if partitioning == "directory":
2854 partitioning = ds.DirectoryPartitioning.discover(
2855 ["part1", "part2"], infer_dictionary=infer_dictionary)
2856 fmt = "{0}/{1}"
2857 null_value = None
2858 else:
2859 if null_fallback:
2860 partitioning = ds.HivePartitioning.discover(
2861 infer_dictionary=infer_dictionary, null_fallback=null_fallback
2862 )
2863 else:
2864 partitioning = ds.HivePartitioning.discover(
2865 infer_dictionary=infer_dictionary)
2866 fmt = "part1={0}/part2={1}"
2867 if null_fallback:
2868 null_value = null_fallback
2869 else:
2870 null_value = "__HIVE_DEFAULT_PARTITION__"
2871
2872 basepath = tempdir / "dataset"
2873 basepath.mkdir()
2874
2875 part_keys1, part_keys2 = partition_keys
2876 for part1 in part_keys1:
2877 for part2 in part_keys2:
2878 path = basepath / \
2879 fmt.format(part1 or null_value, part2 or null_value)
2880 path.mkdir(parents=True)
2881 pq.write_table(table, path / "test.parquet")
2882
2883 dataset = ds.dataset(str(basepath), partitioning=partitioning)
2884
2885 def expected_type(key):
2886 if infer_dictionary:
2887 value_type = pa.string() if isinstance(key, str) else pa.int32()
2888 return pa.dictionary(pa.int32(), value_type)
2889 else:
2890 return pa.string() if isinstance(key, str) else pa.int32()
2891 expected_schema = table.schema.append(
2892 pa.field("part1", expected_type(part_keys1[0]))
2893 ).append(
2894 pa.field("part2", expected_type(part_keys2[0]))
2895 )
2896 assert dataset.schema.equals(expected_schema)
2897
2898
2899@pytest.mark.pandas

Callers

nothing calls this directly

Calls 6

expected_typeFunction · 0.85
equalsMethod · 0.80
formatMethod · 0.45
write_tableMethod · 0.45
appendMethod · 0.45
fieldMethod · 0.45

Tested by

no test coverage detected