Method split

langroid/parsing/parser.py:402–425

(self, docs: List[Document])

        return chunks

    def split(self, docs: List[Document]) -> List[Document]:
        if len(docs) == 0:
            return []
        # create ids in metadata of docs if absent:
        # we need this to distinguish docs later in add_window_ids
        for d in docs:
            if d.metadata.id in [None, ""]:
                d.metadata.id = ObjectRegistry.new_id()
        # some docs are already splits, so don't split them further!
        chunked_docs = [d for d in docs if d.metadata.is_chunk]
        big_docs = [d for d in docs if not d.metadata.is_chunk]
        if len(big_docs) == 0:
            return chunked_docs
        match self.config.splitter:
            case Splitter.MARKDOWN | Splitter.TOKENS:
                big_doc_chunks = self.split_chunk_tokens(big_docs)
            case Splitter.PARA_SENTENCE:
                big_doc_chunks = self.split_para_sentence(big_docs)
            case Splitter.SIMPLE:
                big_doc_chunks = self.split_simple(big_docs)
            case _:
                raise ValueError(f"Unknown splitter: {self.config.splitter}")

        return chunked_docs + big_doc_chunks