(root_path)
| 3913 | |
| 3914 | |
| 3915 | def _create_metadata_file(root_path): |
| 3916 | # create _metadata file from existing parquet dataset |
| 3917 | parquet_paths = list(sorted(root_path.rglob("*.parquet"))) |
| 3918 | schema = pq.ParquetFile(parquet_paths[0]).schema.to_arrow_schema() |
| 3919 | |
| 3920 | metadata_collector = [] |
| 3921 | for path in parquet_paths: |
| 3922 | metadata = pq.ParquetFile(path).metadata |
| 3923 | metadata.set_file_path(str(path.relative_to(root_path))) |
| 3924 | metadata_collector.append(metadata) |
| 3925 | |
| 3926 | metadata_path = root_path / "_metadata" |
| 3927 | pq.write_metadata( |
| 3928 | schema, metadata_path, metadata_collector=metadata_collector |
| 3929 | ) |
| 3930 | return metadata_path |
| 3931 | |
| 3932 | |
| 3933 | def _create_parquet_dataset_partitioned(root_path): |
no test coverage detected