Method split_chunk_tokens

langroid/parsing/parser.py:282–313

(self, docs: List[Document])

        return final_chunks

    def split_chunk_tokens(self, docs: List[Document]) -> List[Document]:
        final_docs = []
        for d in docs:
            if self.config.splitter == Splitter.MARKDOWN:
                chunks = chunk_markdown(
                    d.content,
                    MarkdownChunkConfig(
                        # apply rough adjustment factor to convert from tokens to words,
                        # which is what the markdown chunker uses
                        chunk_size=int(self.config.chunk_size * 0.75),
                        overlap_tokens=int(self.config.overlap * 0.75),
                        variation_percent=self.config.chunk_size_variation,
                        rollup=True,
                    ),
                )
            else:
                chunks = self.chunk_tokens(d.content)
            # note we are ensuring we COPY the document metadata into each chunk,
            # which ensures all chunks of a given doc have same metadata
            # (and in particular same metadata.id, which is important later for
            # add_window_ids)
            chunk_docs = [
                Document(
                    content=c,
                    metadata=d.metadata.model_copy(update=dict(is_chunk=True)),
                )
                for c in chunks
                if c.strip() != ""
            ]
            self.add_window_ids(chunk_docs)
            final_docs += chunk_docs
        return final_docs

    def chunk_tokens(
        self,