Function test_recursive_chunk_enhanced

tests/main/test_md_parser.py:625–709

()

def test_recursive_chunk_enhanced():
    config = MarkdownChunkConfig(
        chunk_size=8,
        overlap_tokens=2,
        variation_percent=0.3,
    )

    # Construct a text with 2 paragraphs, each containing 2 sentences,
    # plus paragraph markers
    paragraph1 = (
        "word1 word2 word3 word4 sentence1.\n"
        "word5 word6 word7 word8 sentence2. PARA1"
    )
    paragraph2 = (
        "cat1 cat2 cat3 cat4 sentence1.\n" "cat5 cat6 cat7 cat8 sentence2. PARA2"
    )

    text = paragraph1 + "\n\n" + paragraph2

    # Now chunk it
    chunks = recursive_chunk(text, config)

    print("\n------------------ ENHANCED CHUNK TEST ------------------")
    for i, c in enumerate(chunks, 1):
        print(f"Chunk {i} ({len(c.split())} words):\n{c}\n")

    # A. Check no chunk splits mid-sentence
    for i, chunk in enumerate(chunks, 1):
        # We expect every sentence boundary to remain intact:
        # "sentence1." or "sentence2." should not be truncated in the middle
        assert (
            "sentence1." in chunk or "sentence2." in chunk or "PARA" in chunk
        ), f"Chunk {i} might have truncated a sentence or lost markers: {chunk}"

    # B. Check paragraph markers do not get merged.
    # We expect that "PARA1" and "PARA2" never appear in the same chunk.
    for i, chunk in enumerate(chunks, 1):
        assert not (
            "PARA1" in chunk and "PARA2" in chunk
        ), "Found both PARA1 and PARA2 in the same chunk!"

    # C. If there's overlap, ensure it's only from chunk (i) to chunk (i+1).
    # A naive check: the last 2 tokens of chunk i = the first 2 tokens of chunk i+1,
    # but chunk i+2 does not contain that same overlap at the start.
    for i in range(len(chunks) - 1):
        chunk_i_tokens = chunks[i].split()
        chunk_i_plus_1_tokens = chunks[i + 1].split()

        overlap_i = chunk_i_tokens[-2:]  # last 2 tokens of chunk i
        start_of_chunk_i_plus_1 = chunk_i_plus_1_tokens[
            :2
        ]  # first 2 tokens of chunk i+1
        assert overlap_i == start_of_chunk_i_plus_1, (
            f"Expected chunk {i+1} to start with overlap tokens from chunk {i}.\n"
            f"Overlap {overlap_i}, got {start_of_chunk_i_plus_1}"
        )

        # Now check chunk (i+2) if it exists