(tempdir)
| 565 | |
| 566 | @pytest.mark.dataset |
| 567 | def test_scan(tempdir): |
| 568 | table = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) |
| 569 | ds.write_dataset(table, tempdir / "dataset", format="parquet") |
| 570 | dataset = ds.dataset(tempdir / "dataset", format="parquet") |
| 571 | decl = Declaration("scan", ScanNodeOptions(dataset)) |
| 572 | result = decl.to_table() |
| 573 | assert result.schema.names == [ |
| 574 | "a", "b", "__fragment_index", "__batch_index", |
| 575 | "__last_in_fragment", "__filename" |
| 576 | ] |
| 577 | assert result.select(["a", "b"]).equals(table) |
| 578 | |
| 579 | # using a filter only does pushdown (depending on file format), not actual filter |
| 580 | |
| 581 | scan_opts = ScanNodeOptions(dataset, filter=field('a') > 1) |
| 582 | decl = Declaration("scan", scan_opts) |
| 583 | # fragment not filtered based on min/max statistics |
| 584 | assert decl.to_table().num_rows == 3 |
| 585 | |
| 586 | scan_opts = ScanNodeOptions(dataset, filter=field('a') > 4) |
| 587 | decl = Declaration("scan", scan_opts) |
| 588 | # full fragment filtered based on min/max statistics |
| 589 | assert decl.to_table().num_rows == 0 |
| 590 | |
| 591 | # projection scan option |
| 592 | |
| 593 | scan_opts = ScanNodeOptions(dataset, columns={"a2": pc.multiply(field("a"), 2)}) |
| 594 | decl = Declaration("scan", scan_opts) |
| 595 | result = decl.to_table() |
| 596 | # "a" is included in the result (needed later on for the actual projection) |
| 597 | assert result["a"].to_pylist() == [1, 2, 3] |
| 598 | # "b" is still included, but without data as it will be removed by the projection |
| 599 | assert pc.all(result["b"].is_null()).as_py() |
nothing calls this directly
no test coverage detected