Class MarkdownChunkConfig

langroid/parsing/md_parser.py:176–191

# Configuration for chunking
class MarkdownChunkConfig(BaseModel):
    chunk_size: int = 200  # desired chunk size in tokens
    overlap_tokens: int = 30  # number of tokens to overlap between chunks
    variation_percent: float = 0.3  # allowed variation
    rollup: bool = True  # whether to roll up chunks
    header_context_sep: str = HEADER_CONTEXT_SEP  # separator for header context

    @field_validator("chunk_size", mode="before")
    @classmethod
    def convert_chunk_size_to_int(cls, v: Any) -> int:
        """Convert chunk_size to int, maintaining backward compatibility
        with Pydantic V1.
        """
        if isinstance(v, float):
            return int(v)
        return int(v)


# A simple tokenizer that counts tokens as whitespace-separated words.