(root_path)
| 3906 | |
| 3907 | |
| 3908 | def _create_metadata_file(root_path): |
| 3909 | # create _metadata file from existing parquet dataset |
| 3910 | parquet_paths = list(sorted(root_path.rglob("*.parquet"))) |
| 3911 | schema = pq.ParquetFile(parquet_paths[0]).schema.to_arrow_schema() |
| 3912 | |
| 3913 | metadata_collector = [] |
| 3914 | for path in parquet_paths: |
| 3915 | metadata = pq.ParquetFile(path).metadata |
| 3916 | metadata.set_file_path(str(path.relative_to(root_path))) |
| 3917 | metadata_collector.append(metadata) |
| 3918 | |
| 3919 | metadata_path = root_path / "_metadata" |
| 3920 | pq.write_metadata( |
| 3921 | schema, metadata_path, metadata_collector=metadata_collector |
| 3922 | ) |
| 3923 | return metadata_path |
| 3924 | |
| 3925 | |
| 3926 | def _create_parquet_dataset_partitioned(root_path): |
no test coverage detected