(tempdir, dataset_reader, pickle_module)
| 1453 | |
| 1454 | @pytest.mark.parquet |
| 1455 | def test_fragments_reconstruct(tempdir, dataset_reader, pickle_module): |
| 1456 | table, dataset = _create_dataset_for_fragments(tempdir) |
| 1457 | |
| 1458 | def assert_yields_projected(fragment, row_slice, |
| 1459 | columns=None, filter=None): |
| 1460 | actual = fragment.to_table( |
| 1461 | schema=table.schema, columns=columns, filter=filter) |
| 1462 | column_names = columns if columns else table.column_names |
| 1463 | assert actual.column_names == column_names |
| 1464 | |
| 1465 | expected = table.slice(*row_slice).select(column_names) |
| 1466 | assert actual.equals(expected) |
| 1467 | |
| 1468 | fragment = list(dataset.get_fragments())[0] |
| 1469 | parquet_format = fragment.format |
| 1470 | |
| 1471 | # test pickle roundtrip |
| 1472 | pickled_fragment = pickle_module.loads(pickle_module.dumps(fragment)) |
| 1473 | assert dataset_reader.to_table( |
| 1474 | pickled_fragment) == dataset_reader.to_table(fragment) |
| 1475 | |
| 1476 | # manually re-construct a fragment, with explicit schema |
| 1477 | new_fragment = parquet_format.make_fragment( |
| 1478 | fragment.path, fragment.filesystem, |
| 1479 | partition_expression=fragment.partition_expression) |
| 1480 | assert dataset_reader.to_table(new_fragment).equals( |
| 1481 | dataset_reader.to_table(fragment)) |
| 1482 | assert_yields_projected(new_fragment, (0, 4)) |
| 1483 | |
| 1484 | # filter / column projection, inspected schema |
| 1485 | new_fragment = parquet_format.make_fragment( |
| 1486 | fragment.path, fragment.filesystem, |
| 1487 | partition_expression=fragment.partition_expression) |
| 1488 | assert_yields_projected(new_fragment, (0, 2), filter=ds.field('f1') < 2) |
| 1489 | |
| 1490 | # filter requiring cast / column projection, inspected schema |
| 1491 | new_fragment = parquet_format.make_fragment( |
| 1492 | fragment.path, fragment.filesystem, |
| 1493 | partition_expression=fragment.partition_expression) |
| 1494 | assert_yields_projected(new_fragment, (0, 2), |
| 1495 | columns=['f1'], filter=ds.field('f1') < 2.0) |
| 1496 | |
| 1497 | # filter on the partition column |
| 1498 | new_fragment = parquet_format.make_fragment( |
| 1499 | fragment.path, fragment.filesystem, |
| 1500 | partition_expression=fragment.partition_expression) |
| 1501 | assert_yields_projected(new_fragment, (0, 4), |
| 1502 | filter=ds.field('part') == 'a') |
| 1503 | |
| 1504 | # Fragments don't contain the partition's columns if not provided to the |
| 1505 | # `to_table(schema=...)` method. |
| 1506 | pattern = (r'No match for FieldRef.Name\(part\) in ' + |
| 1507 | fragment.physical_schema.to_string(False, False, False)) |
| 1508 | with pytest.raises(ValueError, match=pattern): |
| 1509 | new_fragment = parquet_format.make_fragment( |
| 1510 | fragment.path, fragment.filesystem, |
| 1511 | partition_expression=fragment.partition_expression) |
| 1512 | dataset_reader.to_table(new_fragment, filter=ds.field('part') == 'a') |
nothing calls this directly
no test coverage detected