Function chunk_text

nodedb-query/src/chunk_text.rs:82–103

Split text into chunks using the specified strategy. - `chunk_size`: maximum number of **characters** per chunk. - `overlap`: number of characters shared between consecutive chunks. - `strategy`: splitting strategy. Returns a deterministic sequence of chunks. Same input + params → same output.

(
    text: &str,
    chunk_size: usize,
    overlap: usize,
    strategy: ChunkStrategy,
)

///
/// Returns a deterministic sequence of chunks. Same input + params → same output.
pub fn chunk_text(
    text: &str,
    chunk_size: usize,
    overlap: usize,
    strategy: ChunkStrategy,
) -> Result<Vec<TextChunk>, ChunkError> {
    if chunk_size == 0 {
        return Err(ChunkError::InvalidChunkSize);
    }
    if overlap >= chunk_size {
        return Err(ChunkError::OverlapTooLarge);
    }
    if text.is_empty() {
        return Ok(Vec::new());
    }

    match strategy {
        ChunkStrategy::Character => chunk_by_characters(text, chunk_size, overlap),
        ChunkStrategy::Sentence => chunk_by_sentences(text, chunk_size, overlap),
        ChunkStrategy::Paragraph => chunk_by_paragraphs(text, chunk_size, overlap),
    }
}

/// Character-based splitting: advance by `chunk_size - overlap` chars each step.
fn chunk_by_characters(