Function test_text_token_chunking

tests/main/test_parser.py:79–97

(
    chunk_size: int, max_chunks: int, min_chunk_chars: int, discard_chunk_chars: int
)

    ],
)
def test_text_token_chunking(
    chunk_size: int, max_chunks: int, min_chunk_chars: int, discard_chunk_chars: int
):
    cfg = ParsingConfig(
        chunk_size=chunk_size,
        max_chunks=max_chunks,
        min_chunk_chars=min_chunk_chars,
        discard_chunk_chars=discard_chunk_chars,
        token_encoding_model="text-embedding-3-small",
    )

    parser = Parser(cfg)

    text = generate_random_text(60)
    chunks = parser.chunk_tokens(text)

    assert len(chunks) <= max_chunks
    assert all(len(c) >= discard_chunk_chars for c in chunks)
    assert all(parser.num_tokens(c) <= chunk_size + 5 for c in chunks)


def test_extract_content():