(tempdir)
| 474 | |
| 475 | @pytest.mark.dataset |
| 476 | def test_scan(tempdir): |
| 477 | table = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) |
| 478 | ds.write_dataset(table, tempdir / "dataset", format="parquet") |
| 479 | dataset = ds.dataset(tempdir / "dataset", format="parquet") |
| 480 | decl = Declaration("scan", ScanNodeOptions(dataset)) |
| 481 | result = decl.to_table() |
| 482 | assert result.schema.names == [ |
| 483 | "a", "b", "__fragment_index", "__batch_index", |
| 484 | "__last_in_fragment", "__filename" |
| 485 | ] |
| 486 | assert result.select(["a", "b"]).equals(table) |
| 487 | |
| 488 | # using a filter only does pushdown (depending on file format), not actual filter |
| 489 | |
| 490 | scan_opts = ScanNodeOptions(dataset, filter=field('a') > 1) |
| 491 | decl = Declaration("scan", scan_opts) |
| 492 | # fragment not filtered based on min/max statistics |
| 493 | assert decl.to_table().num_rows == 3 |
| 494 | |
| 495 | scan_opts = ScanNodeOptions(dataset, filter=field('a') > 4) |
| 496 | decl = Declaration("scan", scan_opts) |
| 497 | # full fragment filtered based on min/max statistics |
| 498 | assert decl.to_table().num_rows == 0 |
| 499 | |
| 500 | # projection scan option |
| 501 | |
| 502 | scan_opts = ScanNodeOptions(dataset, columns={"a2": pc.multiply(field("a"), 2)}) |
| 503 | decl = Declaration("scan", scan_opts) |
| 504 | result = decl.to_table() |
| 505 | # "a" is included in the result (needed later on for the actual projection) |
| 506 | assert result["a"].to_pylist() == [1, 2, 3] |
| 507 | # "b" is still included, but without data as it will be removed by the projection |
| 508 | assert pc.all(result["b"].is_null()).as_py() |
nothing calls this directly
no test coverage detected