MCPcopy Create free account
hub / github.com/apache/arrow / test_partitioning_factory

Function test_partitioning_factory

python/pyarrow/tests/test_dataset.py:1974–2001  ·  view source on GitHub ↗
(mockfs, pickled, pickle_module)

Source from the content-addressed store, hash-verified

1972@pytest.mark.parametrize(
1973 "pickled", [lambda x, m: x, lambda x, m: m.loads(m.dumps(x))])
1974def test_partitioning_factory(mockfs, pickled, pickle_module):
1975 paths_or_selector = fs.FileSelector('subdir', recursive=True)
1976 format = ds.ParquetFileFormat()
1977
1978 options = ds.FileSystemFactoryOptions('subdir')
1979 partitioning_factory = ds.DirectoryPartitioning.discover(['group', 'key'])
1980 partitioning_factory = pickled(partitioning_factory, pickle_module)
1981 assert isinstance(partitioning_factory, ds.PartitioningFactory)
1982 options.partitioning_factory = partitioning_factory
1983
1984 factory = ds.FileSystemDatasetFactory(
1985 mockfs, paths_or_selector, format, options
1986 )
1987 inspected_schema = factory.inspect()
1988 # i64/f64 from data, group/key from "/1/xxx" and "/2/yyy" paths
1989 expected_schema = pa.schema([
1990 ("i64", pa.int64()),
1991 ("f64", pa.float64()),
1992 ("str", pa.string()),
1993 ("const", pa.int64()),
1994 ("struct", pa.struct({'a': pa.int64(), 'b': pa.string()})),
1995 ("group", pa.int32()),
1996 ("key", pa.string()),
1997 ])
1998 assert inspected_schema.equals(expected_schema)
1999
2000 hive_partitioning_factory = ds.HivePartitioning.discover()
2001 assert isinstance(hive_partitioning_factory, ds.PartitioningFactory)
2002
2003
2004@pytest.mark.parquet

Callers

nothing calls this directly

Calls 6

ParquetFileFormatMethod · 0.80
equalsMethod · 0.80
inspectMethod · 0.45
schemaMethod · 0.45
stringMethod · 0.45

Tested by

no test coverage detected