(tempdir)
| 4545 | |
| 4546 | @pytest.mark.parquet |
| 4547 | def test_write_dataset_min_rows_per_group(tempdir): |
| 4548 | directory = tempdir / 'ds' |
| 4549 | min_rows_per_group = 6 |
| 4550 | max_rows_per_group = 8 |
| 4551 | num_of_columns = 2 |
| 4552 | |
| 4553 | record_sizes = [5, 5, 5, 5, 5, 4, 4, 4, 4, 4] |
| 4554 | |
| 4555 | record_batches = [_generate_data_and_columns(num_of_columns, |
| 4556 | num_of_records) |
| 4557 | for num_of_records in record_sizes] |
| 4558 | |
| 4559 | data_source = directory / "min_rows_group" |
| 4560 | |
| 4561 | ds.write_dataset(data=record_batches, base_dir=data_source, |
| 4562 | min_rows_per_group=min_rows_per_group, |
| 4563 | max_rows_per_group=max_rows_per_group, |
| 4564 | format="parquet") |
| 4565 | |
| 4566 | files_in_dir = os.listdir(data_source) |
| 4567 | for _, f_file in enumerate(files_in_dir): |
| 4568 | f_path = data_source / str(f_file) |
| 4569 | dataset = ds.dataset(f_path, format="parquet") |
| 4570 | table = dataset.to_table() |
| 4571 | batches = table.to_batches() |
| 4572 | |
| 4573 | for id, batch in enumerate(batches): |
| 4574 | rows_per_batch = batch.num_rows |
| 4575 | if id < len(batches) - 1: |
| 4576 | assert rows_per_batch >= min_rows_per_group and \ |
| 4577 | rows_per_batch <= max_rows_per_group |
| 4578 | else: |
| 4579 | assert rows_per_batch <= max_rows_per_group |
| 4580 | |
| 4581 | |
| 4582 | @pytest.mark.parquet |
nothing calls this directly
no test coverage detected