One-time Add documents to both BM25 and semantic index
(self, documents: List[str])
| 470 | return retriever |
| 471 | |
| 472 | def add_documents(self, documents: List[str]) -> bool: |
| 473 | """One-time Add documents to both BM25 and semantic index""" |
| 474 | if not documents: |
| 475 | return |
| 476 | |
| 477 | # Tokenize for BM25 |
| 478 | tokenized_docs = [doc.lower().split() for doc in documents] |
| 479 | self.bm25 = BM25Okapi(tokenized_docs) |
| 480 | |
| 481 | # Create embeddings |
| 482 | self.embeddings = self.model.encode(documents) |
| 483 | self.corpus = documents |
| 484 | doc_idx = 0 |
| 485 | for document in documents: |
| 486 | self.document_ids[document] = doc_idx |
| 487 | doc_idx += 1 |
| 488 | |
| 489 | return True |
| 490 | |
| 491 | def add_document(self, document: str) -> bool: |
| 492 | """Add a single document to the retriever. |
no test coverage detected