Function chunk

src/semble/chunking/core.py:144–161

Chunk source code.

(text: str, language: str, desired_length: int)

def chunk(text: str, language: str, desired_length: int) -> list[ChunkBoundary] | None:
    """Chunk source code."""
    if not text.strip():
        return []

    as_bytes = text.encode("utf-8")
    parser = _cached_get_parser(language)
    if parser is None:
        return None
    root = parser.parse(as_bytes).root_node

    chunks = []
    for chunk_boundary in _merge_node(root, desired_length):
        start_char = len(as_bytes[: chunk_boundary.start].decode("utf-8"))
        end_char = len(as_bytes[: chunk_boundary.end].decode("utf-8"))
        chunks.append(ChunkBoundary(start=start_char, end=end_char))

    return chunks