Function chunk_by_paragraphs

nodedb-query/src/chunk_text.rs:164–175

Paragraph-based splitting: split at double-newline boundaries. If a single paragraph exceeds chunk_size, it falls back to sentence split, then character split if needed.

(
    text: &str,
    chunk_size: usize,
    overlap: usize,
)

/// If a single paragraph exceeds chunk_size, it falls back to sentence split,
/// then character split if needed.
fn chunk_by_paragraphs(
    text: &str,
    chunk_size: usize,
    overlap: usize,
) -> Result<Vec<TextChunk>, ChunkError> {
    let paragraphs = split_paragraphs(text);
    if paragraphs.is_empty() {
        return Ok(Vec::new());
    }

    build_chunks_from_segments(&paragraphs, chunk_size, overlap)
}

/// Split text into sentences at `.` `!` `?` followed by whitespace or end-of-string.
///