(
self,
storage: SavedDatasetStorage,
allow_overwrite: Optional[bool] = False,
timeout: Optional[int] = None,
)
| 103 | return pyarrow.Table.from_pandas(df) |
| 104 | |
| 105 | def persist( |
| 106 | self, |
| 107 | storage: SavedDatasetStorage, |
| 108 | allow_overwrite: Optional[bool] = False, |
| 109 | timeout: Optional[int] = None, |
| 110 | ): |
| 111 | assert isinstance(storage, SavedDatasetFileStorage) |
| 112 | |
| 113 | # Check if the specified location already exists. |
| 114 | if not allow_overwrite and os.path.exists(storage.file_options.uri): |
| 115 | raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri) |
| 116 | absolute_path = FileSource.get_uri_for_file_path( |
| 117 | repo_path=self.repo_path, uri=storage.file_options.uri |
| 118 | ) |
| 119 | |
| 120 | filesystem, path = FileSource.create_filesystem_and_path( |
| 121 | str(absolute_path), |
| 122 | storage.file_options.s3_endpoint_override, |
| 123 | ) |
| 124 | |
| 125 | if path.endswith(".parquet"): |
| 126 | pyarrow.parquet.write_table( |
| 127 | self.to_arrow(), where=path, filesystem=filesystem |
| 128 | ) |
| 129 | else: |
| 130 | # otherwise assume destination is directory |
| 131 | pyarrow.parquet.write_to_dataset( |
| 132 | self.to_arrow(), root_path=path, filesystem=filesystem |
| 133 | ) |
| 134 | |
| 135 | @property |
| 136 | def metadata(self) -> Optional[RetrievalMetadata]: |
no test coverage detected