(tempdir)
| 462 | |
| 463 | @pytest.mark.dataset |
| 464 | def test_scan(tempdir): |
| 465 | table = pa.table({'a': [1, 2, 3], 'b': [4, 5, 6]}) |
| 466 | ds.write_dataset(table, tempdir / "dataset", format="parquet") |
| 467 | dataset = ds.dataset(tempdir / "dataset", format="parquet") |
| 468 | decl = Declaration("scan", ScanNodeOptions(dataset)) |
| 469 | result = decl.to_table() |
| 470 | assert result.schema.names == [ |
| 471 | "a", "b", "__fragment_index", "__batch_index", |
| 472 | "__last_in_fragment", "__filename" |
| 473 | ] |
| 474 | assert result.select(["a", "b"]).equals(table) |
| 475 | |
| 476 | # using a filter only does pushdown (depending on file format), not actual filter |
| 477 | |
| 478 | scan_opts = ScanNodeOptions(dataset, filter=field('a') > 1) |
| 479 | decl = Declaration("scan", scan_opts) |
| 480 | # fragment not filtered based on min/max statistics |
| 481 | assert decl.to_table().num_rows == 3 |
| 482 | |
| 483 | scan_opts = ScanNodeOptions(dataset, filter=field('a') > 4) |
| 484 | decl = Declaration("scan", scan_opts) |
| 485 | # full fragment filtered based on min/max statistics |
| 486 | assert decl.to_table().num_rows == 0 |
| 487 | |
| 488 | # projection scan option |
| 489 | |
| 490 | scan_opts = ScanNodeOptions(dataset, columns={"a2": pc.multiply(field("a"), 2)}) |
| 491 | decl = Declaration("scan", scan_opts) |
| 492 | result = decl.to_table() |
| 493 | # "a" is included in the result (needed later on for the actual projection) |
| 494 | assert result["a"].to_pylist() == [1, 2, 3] |
| 495 | # "b" is still included, but without data as it will be removed by the projection |
| 496 | assert pc.all(result["b"].is_null()).as_py() |
nothing calls this directly
no test coverage detected