(tempdir)
| 4508 | |
| 4509 | @pytest.mark.parquet |
| 4510 | def test_write_dataset_max_rows_per_file(tempdir): |
| 4511 | directory = tempdir / 'ds' |
| 4512 | max_rows_per_file = 10 |
| 4513 | max_rows_per_group = 10 |
| 4514 | num_of_columns = 2 |
| 4515 | num_of_records = 35 |
| 4516 | |
| 4517 | record_batch = _generate_data_and_columns(num_of_columns, |
| 4518 | num_of_records) |
| 4519 | |
| 4520 | ds.write_dataset(record_batch, directory, format="parquet", |
| 4521 | max_rows_per_file=max_rows_per_file, |
| 4522 | max_rows_per_group=max_rows_per_group) |
| 4523 | |
| 4524 | files_in_dir = os.listdir(directory) |
| 4525 | |
| 4526 | # number of partitions with max_rows and the partition with the remainder |
| 4527 | expected_partitions = num_of_records // max_rows_per_file + 1 |
| 4528 | |
| 4529 | # test whether the expected amount of files are written |
| 4530 | assert len(files_in_dir) == expected_partitions |
| 4531 | |
| 4532 | # compute the number of rows per each file written |
| 4533 | result_row_combination = [] |
| 4534 | for _, f_file in enumerate(files_in_dir): |
| 4535 | f_path = directory / str(f_file) |
| 4536 | dataset = ds.dataset(f_path, format="parquet") |
| 4537 | result_row_combination.append(dataset.to_table().shape[0]) |
| 4538 | |
| 4539 | # test whether the generated files have the expected number of rows |
| 4540 | assert expected_partitions == len(result_row_combination) |
| 4541 | assert num_of_records == sum(result_row_combination) |
| 4542 | assert all(file_rowcount <= max_rows_per_file |
| 4543 | for file_rowcount in result_row_combination) |
| 4544 | |
| 4545 | |
| 4546 | @pytest.mark.parquet |
nothing calls this directly
no test coverage detected