MCPcopy Create free account
hub / github.com/feast-dev/feast / persist

Method persist

sdk/python/feast/infra/offline_stores/dask.py:105–133  ·  view source on GitHub ↗
(
        self,
        storage: SavedDatasetStorage,
        allow_overwrite: Optional[bool] = False,
        timeout: Optional[int] = None,
    )

Source from the content-addressed store, hash-verified

103 return pyarrow.Table.from_pandas(df)
104
105 def persist(
106 self,
107 storage: SavedDatasetStorage,
108 allow_overwrite: Optional[bool] = False,
109 timeout: Optional[int] = None,
110 ):
111 assert isinstance(storage, SavedDatasetFileStorage)
112
113 # Check if the specified location already exists.
114 if not allow_overwrite and os.path.exists(storage.file_options.uri):
115 raise SavedDatasetLocationAlreadyExists(location=storage.file_options.uri)
116 absolute_path = FileSource.get_uri_for_file_path(
117 repo_path=self.repo_path, uri=storage.file_options.uri
118 )
119
120 filesystem, path = FileSource.create_filesystem_and_path(
121 str(absolute_path),
122 storage.file_options.s3_endpoint_override,
123 )
124
125 if path.endswith(".parquet"):
126 pyarrow.parquet.write_table(
127 self.to_arrow(), where=path, filesystem=filesystem
128 )
129 else:
130 # otherwise assume destination is directory
131 pyarrow.parquet.write_to_dataset(
132 self.to_arrow(), root_path=path, filesystem=filesystem
133 )
134
135 @property
136 def metadata(self) -> Optional[RetrievalMetadata]:

Callers 10

evaluate_funcMethod · 0.45
evaluate_offline_jobMethod · 0.45
_run_dask_field_mappingFunction · 0.45
_field_mappingFunction · 0.45
_mergeFunction · 0.45
_normalize_timestampFunction · 0.45
_filter_ttlFunction · 0.45
_drop_duplicatesFunction · 0.45
_drop_columnsFunction · 0.45

Calls 5

get_uri_for_file_pathMethod · 0.80
write_tableMethod · 0.80
to_arrowMethod · 0.45

Tested by

no test coverage detected