Function chunk_text

explainshell/extraction/llm/text.py:196–301

Split text into numbered chunks at section boundaries. Each chunk gets line numbers corresponding to the original text. If the text is small enough, returns a single chunk. A preamble (NAME/SYNOPSIS/DESCRIPTION intro) is prepended to each chunk beyond the first so the model has cont

(text: str)

def chunk_text(text: str) -> list[str]:
    """Split text into numbered chunks at section boundaries.

    Each chunk gets line numbers corresponding to the original text.
    If the text is small enough, returns a single chunk.
    A preamble (NAME/SYNOPSIS/DESCRIPTION intro) is prepended to each
    chunk beyond the first so the model has context.

    When a single section exceeds the chunk size, it is sub-split at
    paragraph (blank-line) boundaries.
    """
    numbered_full, _ = number_lines(text)
    if len(numbered_full) <= CHUNK_SIZE_CHARS:
        return [numbered_full]

    sections = _split_sections(text)
    preamble = _build_preamble(text)
    preamble_text = ""
    if preamble:
        preamble_text = (
            "[Context — this is a continuation of the same man page]\n\n"
            + preamble
            + "\n\n---\n\n"
        )
    budget = CHUNK_SIZE_CHARS - len(preamble_text)

    total_lines = text.count("\n") + 1
    width = len(str(total_lines))

    def _number_block(start_line: int, block_text: str) -> str:
        lines = block_text.split("\n")
        numbered = []
        for j, line in enumerate(lines):
            lineno = start_line + j
            numbered.append(f"{lineno:>{width}}| {line}")
        return "\n".join(numbered)

    def _split_by_lines(start_line: int, block_text: str) -> list[tuple[int, str]]:
        """Last-resort split: cut at line boundaries to fit budget."""
        lines = block_text.split("\n")
        result: list[tuple[int, str]] = []
        cur_lines: list[str] = []
        cur_start = start_line
        for line in lines:
            candidate = "\n".join(cur_lines + [line])
            if len(_number_block(cur_start, candidate)) > budget and cur_lines:
                result.append((cur_start, "\n".join(cur_lines)))
                cur_start += len(cur_lines)
                cur_lines = []
            cur_lines.append(line)
        if cur_lines:
            result.append((cur_start, "\n".join(cur_lines)))
        return result

    blocks: list[tuple[int, str]] = []
    for start_line, section_text in sections:
        numbered = _number_block(start_line, section_text)
        if len(numbered) <= budget: