Method split_text

core/src/text_splitter.rs:217–306

(&self, text: &str)

impl TextSplitterTrait for CharacterSplitter {
    fn split_text(&self, text: &str) -> GraphBitResult<Vec<TextChunk>> {
        if text.is_empty() {
            return Ok(Vec::new());
        }

        let mut chunks = Vec::new();
        let mut chunk_index = 0;

        // Build a mapping of character index to byte index
        let char_to_byte: Vec<(usize, usize)> = text
            .char_indices()
            .enumerate()
            .map(|(char_idx, (byte_idx, _))| (char_idx, byte_idx))
            .collect();

        // Add the end position
        let total_chars = text.chars().count();
        let mut char_to_byte_map = char_to_byte;
        char_to_byte_map.push((total_chars, text.len()));

        let mut char_start = 0;

        while char_start < total_chars {
            let mut char_end = (char_start + self.chunk_size).min(total_chars);

            // Get byte positions
            let mut byte_start = char_to_byte_map[char_start].1;
            let mut byte_end = char_to_byte_map[char_end].1;

            // Preserve word boundaries if enabled
            if self.config.preserve_word_boundaries && char_end < total_chars {
                // Look for the last whitespace in the chunk
                let chunk_text = &text[byte_start..byte_end];
                if let Some(last_space_pos) = chunk_text.rfind(char::is_whitespace) {
                    // Adjust to the position after the whitespace
                    byte_end = byte_start
                        + last_space_pos
                        + chunk_text[last_space_pos..]
                            .chars()
                            .next()
                            .unwrap()
                            .len_utf8();

                    // Find the corresponding character position
                    for i in (char_start..char_end).rev() {
                        if char_to_byte_map[i + 1].1 == byte_end {
                            char_end = i + 1;
                            break;
                        }
                    }
                }
            }

            let chunk_content = &text[byte_start..byte_end];
            let content = if self.config.trim_whitespace {
                chunk_content.trim()
            } else {
                chunk_content