Function test_parser

tests/main/test_parser.py:20–64

(
    splitter: Splitter,
    chunk_size: int,
    max_chunks: int,
    min_chunk_chars: int,
    discard_chunk_chars: int,
)

    ],
)
def test_parser(
    splitter: Splitter,
    chunk_size: int,
    max_chunks: int,
    min_chunk_chars: int,
    discard_chunk_chars: int,
):
    cfg = ParsingConfig(
        splitter=splitter,
        n_neighbor_ids=2,
        chunk_size_variation=0.2,
        chunk_size=chunk_size,
        max_chunks=max_chunks,
        separators=["."],
        min_chunk_chars=min_chunk_chars,
        discard_chunk_chars=discard_chunk_chars,
        token_encoding_model="text-embedding-3-small",
    )

    parser = Parser(cfg)
    docs = [
        Document(content=generate_random_text(500), metadata={"id": i})
        for i in range(5)
    ]

    split_docs = parser.split(docs)

    chunk_size_upper_bound = (
        chunk_size * (1 + cfg.chunk_size_variation)
        if splitter == Splitter.MARKDOWN
        else chunk_size + 5
    )
    assert all(
        parser.num_tokens(d.content) <= chunk_size_upper_bound for d in split_docs
    )
    assert len(split_docs) <= max_chunks * len(docs)
    assert all(len(d.content) >= discard_chunk_chars for d in split_docs)
    assert all(d.metadata.is_chunk for d in split_docs)

    # test neighbor chunks
    doc = Document(content=generate_random_text(500), metadata={"id": 0})
    chunks = parser.split([doc])
    n = len(chunks)
    if n > 2 * cfg.n_neighbor_ids + 1:
        assert len(chunks[n // 2].metadata.window_ids) == 2 * cfg.n_neighbor_ids + 1


def length_fn(text):