MCPcopy Create free account
hub / github.com/apache/arrow / test_fragments

Function test_fragments

python/pyarrow/tests/test_dataset.py:1408–1437  ·  view source on GitHub ↗
(tempdir, dataset_reader)

Source from the content-addressed store, hash-verified

1406
1407@pytest.mark.parquet
1408def test_fragments(tempdir, dataset_reader):
1409 table, dataset = _create_dataset_for_fragments(tempdir)
1410
1411 # list fragments
1412 fragments = list(dataset.get_fragments())
1413 assert len(fragments) == 2
1414 f = fragments[0]
1415
1416 physical_names = ['f1', 'f2']
1417 # file's schema does not include partition column
1418 assert f.physical_schema.names == physical_names
1419 assert f.format.inspect(f.path, f.filesystem) == f.physical_schema
1420 assert f.partition_expression.equals(ds.field('part') == 'a')
1421
1422 # By default, the partition column is not part of the schema.
1423 result = dataset_reader.to_table(f)
1424 assert result.column_names == physical_names
1425 assert result.equals(table.remove_column(2).slice(0, 4))
1426
1427 # scanning fragment includes partition columns when given the proper
1428 # schema.
1429 result = dataset_reader.to_table(f, schema=dataset.schema)
1430 assert result.column_names == ['f1', 'f2', 'part']
1431 assert result.equals(table.slice(0, 4))
1432 assert f.physical_schema == result.schema.remove(2)
1433
1434 # scanning fragments follow filter predicate
1435 result = dataset_reader.to_table(
1436 f, schema=dataset.schema, filter=ds.field('f1') < 2)
1437 assert result.column_names == ['f1', 'f2', 'part']
1438
1439
1440@pytest.mark.pandas

Callers

nothing calls this directly

Calls 9

listFunction · 0.85
lenFunction · 0.85
equalsMethod · 0.80
inspectMethod · 0.45
fieldMethod · 0.45
to_tableMethod · 0.45
sliceMethod · 0.45
remove_columnMethod · 0.45

Tested by

no test coverage detected