Method split_para_sentence

langroid/parsing/parser.py:229–258

(self, docs: List[Document])

        return final_docs

    def split_para_sentence(self, docs: List[Document]) -> List[Document]:
        chunks = docs
        while True:
            un_splittables = 0
            split_chunks = []
            for c in chunks:
                if c.content.strip() == "":
                    continue
                if self.num_tokens(c.content) <= 1.3 * self.config.chunk_size:
                    # small chunk: no need to split
                    split_chunks.append(c)
                    continue
                splits = self._split_para_sentence_once([c])
                un_splittables += len(splits) == 1
                split_chunks += splits
            if len(split_chunks) == len(chunks):
                if un_splittables > 0:
                    max_len = max([self.num_tokens(p.content) for p in chunks])
                    logger.warning(
                        f"""
                        Unable to split {un_splittables} chunks
                        using chunk_size = {self.config.chunk_size}.
                        Max chunk size is {max_len} tokens.
                        """
                    )
                break  # we won't be able to shorten them with current settings
            chunks = split_chunks.copy()

        self.add_window_ids(chunks)
        return chunks

    def _split_para_sentence_once(self, docs: List[Document]) -> List[Document]:
        final_chunks = []