(tempdir)
| 471 | |
| 472 | @pytest.mark.pandas |
| 473 | def test_multi_dataset_metadata(tempdir): |
| 474 | filenames = ["ARROW-1983-dataset.0", "ARROW-1983-dataset.1"] |
| 475 | metapath = str(tempdir / "_metadata") |
| 476 | |
| 477 | # create a test dataset |
| 478 | df = pd.DataFrame({ |
| 479 | 'one': [1, 2, 3], |
| 480 | 'two': [-1, -2, -3], |
| 481 | 'three': [[1, 2], [2, 3], [3, 4]], |
| 482 | }) |
| 483 | table = pa.Table.from_pandas(df) |
| 484 | |
| 485 | # write dataset twice and collect/merge metadata |
| 486 | _meta = None |
| 487 | for filename in filenames: |
| 488 | meta = [] |
| 489 | pq.write_table(table, str(tempdir / filename), |
| 490 | metadata_collector=meta) |
| 491 | meta[0].set_file_path(filename) |
| 492 | if _meta is None: |
| 493 | _meta = meta[0] |
| 494 | else: |
| 495 | _meta.append_row_groups(meta[0]) |
| 496 | |
| 497 | # Write merged metadata-only file |
| 498 | with open(metapath, "wb") as f: |
| 499 | _meta.write_metadata_file(f) |
| 500 | |
| 501 | # Read back the metadata |
| 502 | meta = pq.read_metadata(metapath) |
| 503 | md = meta.to_dict() |
| 504 | _md = _meta.to_dict() |
| 505 | for key in _md: |
| 506 | if key != 'serialized_size': |
| 507 | assert _md[key] == md[key] |
| 508 | assert _md['num_columns'] == 3 |
| 509 | assert _md['num_rows'] == 6 |
| 510 | assert _md['num_row_groups'] == 2 |
| 511 | assert _md['serialized_size'] == 0 |
| 512 | assert md['serialized_size'] > 0 |
| 513 | |
| 514 | |
| 515 | def test_metadata_hashing(tempdir): |
nothing calls this directly
no test coverage detected