Creates a simple (flat files, no nested partitioning) Parquet dataset
(root_path)
| 3792 | |
| 3793 | |
| 3794 | def _create_parquet_dataset_simple(root_path): |
| 3795 | """ |
| 3796 | Creates a simple (flat files, no nested partitioning) Parquet dataset |
| 3797 | """ |
| 3798 | |
| 3799 | metadata_collector = [] |
| 3800 | |
| 3801 | for i in range(4): |
| 3802 | table = pa.table({'f1': [i] * 10, 'f2': [random.random() for _ in range(10)]}) |
| 3803 | pq.write_to_dataset( |
| 3804 | table, str(root_path), metadata_collector=metadata_collector |
| 3805 | ) |
| 3806 | |
| 3807 | metadata_path = str(root_path / '_metadata') |
| 3808 | # write _metadata file |
| 3809 | pq.write_metadata( |
| 3810 | table.schema, metadata_path, |
| 3811 | metadata_collector=metadata_collector |
| 3812 | ) |
| 3813 | return metadata_path, table |
| 3814 | |
| 3815 | |
| 3816 | @pytest.mark.parquet |
no outgoing calls
no test coverage detected