(tempdir)
| 4581 | |
| 4582 | @pytest.mark.parquet |
| 4583 | def test_write_dataset_max_rows_per_group(tempdir): |
| 4584 | directory = tempdir / 'ds' |
| 4585 | max_rows_per_group = 18 |
| 4586 | num_of_columns = 2 |
| 4587 | num_of_records = 30 |
| 4588 | |
| 4589 | record_batch = _generate_data_and_columns(num_of_columns, |
| 4590 | num_of_records) |
| 4591 | |
| 4592 | data_source = directory / "max_rows_group" |
| 4593 | |
| 4594 | ds.write_dataset(data=record_batch, base_dir=data_source, |
| 4595 | max_rows_per_group=max_rows_per_group, |
| 4596 | format="parquet") |
| 4597 | |
| 4598 | files_in_dir = os.listdir(data_source) |
| 4599 | batched_data = [] |
| 4600 | for f_file in files_in_dir: |
| 4601 | f_path = data_source / str(f_file) |
| 4602 | dataset = ds.dataset(f_path, format="parquet") |
| 4603 | table = dataset.to_table() |
| 4604 | batches = table.to_batches() |
| 4605 | for batch in batches: |
| 4606 | batched_data.append(batch.num_rows) |
| 4607 | |
| 4608 | assert batched_data == [18, 12] |
| 4609 | |
| 4610 | |
| 4611 | @pytest.mark.parquet |
nothing calls this directly
no test coverage detected