MCPcopy Create free account
hub / github.com/apache/arrow / test_filesystem_factory

Function test_filesystem_factory

python/pyarrow/tests/test_dataset.py:1147–1216  ·  view source on GitHub ↗
(mockfs, paths_or_selector, pre_buffer)

Source from the content-addressed store, hash-verified

1145@pytest.mark.parametrize('pre_buffer', [False, True])
1146@pytest.mark.parquet
1147def test_filesystem_factory(mockfs, paths_or_selector, pre_buffer):
1148 format = ds.ParquetFileFormat(
1149 read_options=ds.ParquetReadOptions(dictionary_columns={"str"}),
1150 pre_buffer=pre_buffer
1151 )
1152
1153 options = ds.FileSystemFactoryOptions('subdir')
1154 options.partitioning = ds.DirectoryPartitioning(
1155 pa.schema([
1156 pa.field('group', pa.int32()),
1157 pa.field('key', pa.string())
1158 ])
1159 )
1160 assert options.partition_base_dir == 'subdir'
1161 assert options.selector_ignore_prefixes == ['.', '_']
1162 assert options.exclude_invalid_files is False
1163
1164 factory = ds.FileSystemDatasetFactory(
1165 mockfs, paths_or_selector, format, options
1166 )
1167 inspected_schema = factory.inspect()
1168
1169 assert factory.inspect().equals(pa.schema([
1170 pa.field('i64', pa.int64()),
1171 pa.field('f64', pa.float64()),
1172 pa.field('str', pa.dictionary(pa.int32(), pa.string())),
1173 pa.field('const', pa.int64()),
1174 pa.field('struct', pa.struct({'a': pa.int64(),
1175 'b': pa.string()})),
1176 pa.field('group', pa.int32()),
1177 pa.field('key', pa.string()),
1178 ]), check_metadata=False)
1179
1180 assert isinstance(factory.inspect_schemas(), list)
1181 assert isinstance(factory.finish(inspected_schema),
1182 ds.FileSystemDataset)
1183 assert factory.root_partition.equals(ds.scalar(True))
1184
1185 dataset = factory.finish()
1186 assert isinstance(dataset, ds.FileSystemDataset)
1187
1188 scanner = dataset.scanner()
1189 expected_i64 = pa.array([0, 1, 2, 3, 4], type=pa.int64())
1190 expected_f64 = pa.array([0, 1, 2, 3, 4], type=pa.float64())
1191 expected_str = pa.DictionaryArray.from_arrays(
1192 pa.array([0, 1, 2, 3, 4], type=pa.int32()),
1193 pa.array("0 1 2 3 4".split(), type=pa.string())
1194 )
1195 expected_struct = pa.array([{'a': i % 3, 'b': str(i % 3)}
1196 for i in range(5)])
1197 iterator = scanner.scan_batches()
1198 for (batch, fragment), group, key in zip(iterator, [1, 2], ['xxx', 'yyy']):
1199 expected_group = pa.array([group] * 5, type=pa.int32())
1200 expected_key = pa.array([key] * 5, type=pa.string())
1201 expected_const = pa.array([group - 1] * 5, type=pa.int64())
1202 # Can't compare or really introspect expressions from Python
1203 assert fragment.partition_expression is not None
1204 assert batch.num_columns == 7

Callers

nothing calls this directly

Calls 15

lenFunction · 0.85
ParquetFileFormatMethod · 0.80
DirectoryPartitioningMethod · 0.80
equalsMethod · 0.80
scannerMethod · 0.80
schemaMethod · 0.45
fieldMethod · 0.45
stringMethod · 0.45
inspectMethod · 0.45
dictionaryMethod · 0.45
finishMethod · 0.45

Tested by

no test coverage detected