Function test_recursive_chunk

tests/main/test_md_parser.py:561–622

Tests that the chunker respects paragraph boundaries when possible, then sentence boundaries, and only splits sentences when no other option is possible under the given config.

(chunk_size, overlap_tokens, variation_percent)

    ],
)
def test_recursive_chunk(chunk_size, overlap_tokens, variation_percent):
    """
    Tests that the chunker respects paragraph boundaries when possible,
    then sentence boundaries, and only splits sentences when no other option
    is possible under the given config.
    """
    # Generate some text with 2 paragraphs, each having 3 sentences of 10 words.
    # ~ Each paragraph => 3 sentences =>
    #    each sentence has ~10 words => ~30 words per paragraph.
    # So total words ~60. This helps us see chunking behavior across boundaries.
    paragraph1 = generate_paragraph(
        sentence_count=3, words_per_sentence=10, paragraph_id=1
    )
    paragraph2 = generate_paragraph(
        sentence_count=3, words_per_sentence=10, paragraph_id=2
    )

    # Combine paragraphs with a double-newline
    text = paragraph1 + "\n\n" + paragraph2

    config = MarkdownChunkConfig(
        chunk_size=chunk_size,
        overlap_tokens=overlap_tokens,
        variation_percent=variation_percent,
    )

    chunks = recursive_chunk(text, config)

    # Print a condensed view for manual inspection
    print("\n===================================")
    print(
        f"Config: chunk_size={chunk_size}, "
        f"overlap_tokens={overlap_tokens}, "
        f"variation_percent={variation_percent}\n"
    )
    print("Generated Text (first 30 words):")
    print(" ".join(text.split()[:30]), "...")
    print("\nChunks:")
    print(condensed_chunk_view(chunks, max_words=5))
    print("===================================\n")

    # Basic asserts:
    # 1. No chunk should exceed the upper bound in terms of word count
    upper_bound = chunk_size * (1 + variation_percent)
    for i, chunk in enumerate(chunks):
        word_count_in_chunk = len(chunk.split())
        assert word_count_in_chunk <= upper_bound + 5, (
            f"Chunk {i+1} has {word_count_in_chunk} words, "
            f"exceeds upper bound (~{upper_bound:.1f})."
        )

    # 2. Check that chunking doesn't produce empty chunks
    for i, chunk in enumerate(chunks):
        assert chunk.strip(), f"Chunk {i+1} is empty!"

    # 3. (Optional) If chunk_size is >= total words, we expect exactly 1 chunk
    total_words = len(text.split())
    if total_words <= chunk_size * (1 + variation_percent):