Function _chunk_text_with_overlap

agents/reading-agent/main.py:3153–3211

(
    text: str,
    section: str,
    max_chars: int = READING_REPORT_CHUNK_CHARS,
    overlap: int = READING_REPORT_CHUNK_OVERLAP,
)

def _chunk_text_with_overlap(
    text: str,
    section: str,
    max_chars: int = READING_REPORT_CHUNK_CHARS,
    overlap: int = READING_REPORT_CHUNK_OVERLAP,
) -> List[Dict[str, Any]]:
    normalized = _clean_pdf_evidence_text(text)
    if not normalized:
        return []

    if max_chars <= 0:
        max_chars = READING_REPORT_CHUNK_CHARS
    overlap = max(0, min(overlap, max_chars // 2 if max_chars > 1 else 0))

    chunks: List[Dict[str, Any]] = []
    start = 0
    text_length = len(normalized)
    while start < text_length:
        end = min(text_length, start + max_chars)
        if end < text_length:
            window = normalized[start:end]
            boundary_candidates = [
                window.rfind("\n\n"),
                window.rfind(". "),
                window.rfind("? "),
                window.rfind("! "),
                window.rfind("; "),
                window.rfind("。"),
                window.rfind("！"),
                window.rfind("？"),
                window.rfind("；"),
            ]
            boundary = max(boundary_candidates)
            if boundary >= max_chars // 2:
                boundary_len = 2 if window[boundary:boundary + 2] in {". ", "? ", "! ", "; ", "\n\n"} else 1
                end = start + boundary + boundary_len

        chunk_text = normalized[start:end].strip()
        chunk_text = _clean_pdf_evidence_text(chunk_text)
        if chunk_text and not _is_noisy_pdf_evidence_text(chunk_text):
            chunks.append(
                {
                    "section": _clean_text(section) or "full_text",
                    "text": chunk_text,
                    "start": start,
                    "end": end,
                    "chunk_index": len(chunks),
                }
            )

        if end >= text_length:
            break

        next_start = max(end - overlap, start + 1)
        if next_start <= start:
            next_start = end
        start = next_start

    return chunks