Save retriever state to disk
(self, retriever_cache_file: str, retriever_cache_embeddings_file: str)
| 418 | |
| 419 | |
| 420 | def save(self, retriever_cache_file: str, retriever_cache_embeddings_file: str): |
| 421 | """Save retriever state to disk""" |
| 422 | |
| 423 | # Save embeddings using numpy |
| 424 | if self.embeddings is not None: |
| 425 | np.save(retriever_cache_embeddings_file, self.embeddings) |
| 426 | |
| 427 | # Save everything else using pickle |
| 428 | state = { |
| 429 | 'alpha': self.alpha, |
| 430 | 'bm25': self.bm25, |
| 431 | 'corpus': self.corpus, |
| 432 | 'document_ids': self.document_ids, |
| 433 | 'model_name': 'all-MiniLM-L6-v2' # Default value for model name |
| 434 | } |
| 435 | |
| 436 | # Try to get the actual model name if possible |
| 437 | try: |
| 438 | state['model_name'] = self.model.get_config_dict()['model_name'] |
| 439 | except (AttributeError, KeyError): |
| 440 | pass |
| 441 | |
| 442 | with open(retriever_cache_file, 'wb') as f: |
| 443 | pickle.dump(state, f) |
| 444 | |
| 445 | @classmethod |
| 446 | def load(cls, retriever_cache_file: str, retriever_cache_embeddings_file: str): |
no test coverage detected