Method add_documents

langroid/vector_store/lancedb.py:196–258

(self, documents: Sequence[Document])

            self.delete_collection(collection_name)

    def add_documents(self, documents: Sequence[Document]) -> None:
        super().maybe_add_ids(documents)
        colls = self.list_collections(empty=True)
        if len(documents) == 0:
            return
        embedding_vecs = self.embedding_fn([doc.content for doc in documents])
        coll_name = self.config.collection_name
        if coll_name is None:
            raise ValueError("No collection name set, cannot ingest docs")
        # self._maybe_set_doc_class_schema(documents[0])
        table_exists = False
        if (
            coll_name in colls
            and self.client.open_table(coll_name).head(1).shape[0] > 0
        ):
            # collection exists and  is not empty:
            # if replace_collection is True, we'll overwrite the existing collection,
            # else we'll append to it.
            if self.config.replace_collection:
                self.client.drop_table(coll_name)
            else:
                table_exists = True

        ids = [str(d.id()) for d in documents]
        # don't insert all at once, batch in chunks of b,
        # else we get an API error
        b = self.config.batch_size

        def make_batches() -> Generator[List[Dict[str, Any]], None, None]:
            for i in range(0, len(ids), b):
                batch = [
                    dict(
                        id=ids[i + j],
                        vector=embedding_vecs[i + j],
                        **doc.model_dump(),
                    )
                    for j, doc in enumerate(documents[i : i + b])
                ]
                yield batch

        try:
            if table_exists:
                tbl = self.client.open_table(coll_name)
                tbl.add(make_batches())
            else:
                batch_gen = make_batches()
                batch = next(batch_gen)
                # use first batch to create table...
                tbl = self.client.create_table(
                    coll_name,
                    data=batch,
                    mode="create",
                )
                # ... and add the rest
                tbl.add(batch_gen)
        except Exception as e:
            logger.error(
                f"""