MCPcopy Create free account
hub / github.com/apache/arrow / test_fragments_reconstruct

Function test_fragments_reconstruct

python/pyarrow/tests/test_dataset.py:1455–1512  ·  view source on GitHub ↗
(tempdir, dataset_reader, pickle_module)

Source from the content-addressed store, hash-verified

1453
1454@pytest.mark.parquet
1455def test_fragments_reconstruct(tempdir, dataset_reader, pickle_module):
1456 table, dataset = _create_dataset_for_fragments(tempdir)
1457
1458 def assert_yields_projected(fragment, row_slice,
1459 columns=None, filter=None):
1460 actual = fragment.to_table(
1461 schema=table.schema, columns=columns, filter=filter)
1462 column_names = columns if columns else table.column_names
1463 assert actual.column_names == column_names
1464
1465 expected = table.slice(*row_slice).select(column_names)
1466 assert actual.equals(expected)
1467
1468 fragment = list(dataset.get_fragments())[0]
1469 parquet_format = fragment.format
1470
1471 # test pickle roundtrip
1472 pickled_fragment = pickle_module.loads(pickle_module.dumps(fragment))
1473 assert dataset_reader.to_table(
1474 pickled_fragment) == dataset_reader.to_table(fragment)
1475
1476 # manually re-construct a fragment, with explicit schema
1477 new_fragment = parquet_format.make_fragment(
1478 fragment.path, fragment.filesystem,
1479 partition_expression=fragment.partition_expression)
1480 assert dataset_reader.to_table(new_fragment).equals(
1481 dataset_reader.to_table(fragment))
1482 assert_yields_projected(new_fragment, (0, 4))
1483
1484 # filter / column projection, inspected schema
1485 new_fragment = parquet_format.make_fragment(
1486 fragment.path, fragment.filesystem,
1487 partition_expression=fragment.partition_expression)
1488 assert_yields_projected(new_fragment, (0, 2), filter=ds.field('f1') < 2)
1489
1490 # filter requiring cast / column projection, inspected schema
1491 new_fragment = parquet_format.make_fragment(
1492 fragment.path, fragment.filesystem,
1493 partition_expression=fragment.partition_expression)
1494 assert_yields_projected(new_fragment, (0, 2),
1495 columns=['f1'], filter=ds.field('f1') < 2.0)
1496
1497 # filter on the partition column
1498 new_fragment = parquet_format.make_fragment(
1499 fragment.path, fragment.filesystem,
1500 partition_expression=fragment.partition_expression)
1501 assert_yields_projected(new_fragment, (0, 4),
1502 filter=ds.field('part') == 'a')
1503
1504 # Fragments don't contain the partition's columns if not provided to the
1505 # `to_table(schema=...)` method.
1506 pattern = (r'No match for FieldRef.Name\(part\) in ' +
1507 fragment.physical_schema.to_string(False, False, False))
1508 with pytest.raises(ValueError, match=pattern):
1509 new_fragment = parquet_format.make_fragment(
1510 fragment.path, fragment.filesystem,
1511 partition_expression=fragment.partition_expression)
1512 dataset_reader.to_table(new_fragment, filter=ds.field('part') == 'a')

Callers

nothing calls this directly

Calls 9

listFunction · 0.85
assert_yields_projectedFunction · 0.85
loadsMethod · 0.80
dumpsMethod · 0.80
equalsMethod · 0.80
to_tableMethod · 0.45
fieldMethod · 0.45
to_stringMethod · 0.45

Tested by

no test coverage detected