(collection_name, question, urls)
| 54 | keys: Dict[str, any] |
| 55 | |
| 56 | def create_collection(collection_name, question, urls): |
| 57 | print("---Got Results---") |
| 58 | docs = [WebBaseLoader(url).load() for url in urls] |
| 59 | docs_list = [item for sublist in docs for item in sublist] |
| 60 | text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder( |
| 61 | chunk_size=1000, chunk_overlap=0 |
| 62 | ) |
| 63 | doc_splits = text_splitter.split_documents(docs_list) |
| 64 | print("---CREATING NEW DOCUMENTS---") |
| 65 | vectorstore = Chroma.from_documents( |
| 66 | documents=doc_splits, |
| 67 | collection_name=collection_name, |
| 68 | embedding=OpenAIEmbeddings(), |
| 69 | ) |
| 70 | create_record(collection_name, urls) |
| 71 | print(f"Collection '{collection_name}' created successfully.") |
| 72 | return vectorstore.as_retriever() |
| 73 | |
| 74 | def retrieve_documents(collection_name, question): |
| 75 | print("---RETRIEVING OLD DOCUMENTS---") |
no test coverage detected