Build chunks from pre-split segments, respecting chunk_size and overlap. Segments that exceed chunk_size are split by characters as a fallback.
(
segments: &[(usize, String)],
chunk_size: usize,
overlap: usize,
)
| 266 | /// |
| 267 | /// Segments that exceed chunk_size are split by characters as a fallback. |
| 268 | fn build_chunks_from_segments( |
| 269 | segments: &[(usize, String)], |
| 270 | chunk_size: usize, |
| 271 | overlap: usize, |
| 272 | ) -> Result<Vec<TextChunk>, ChunkError> { |
| 273 | let mut chunks = Vec::new(); |
| 274 | let mut current_text = String::new(); |
| 275 | let mut current_start: Option<usize> = None; |
| 276 | let mut index = 0usize; |
| 277 | |
| 278 | for (seg_offset, seg_text) in segments { |
| 279 | let seg_chars = seg_text.chars().count(); |
| 280 | |
| 281 | // If a single segment exceeds chunk_size, split it by characters. |
| 282 | if seg_chars > chunk_size { |
| 283 | // Flush current buffer first. |
| 284 | if !current_text.is_empty() { |
| 285 | let start = current_start.unwrap_or(0); |
| 286 | let end = start + current_text.chars().count(); |
| 287 | chunks.push(TextChunk { |
| 288 | index, |
| 289 | start, |
| 290 | end, |
| 291 | text: std::mem::take(&mut current_text), |
| 292 | }); |
| 293 | index += 1; |
| 294 | current_start = None; |
| 295 | } |
| 296 | |
| 297 | // Character-split the oversized segment. |
| 298 | let sub_chunks = chunk_by_characters(seg_text, chunk_size, overlap)?; |
| 299 | for sub in sub_chunks { |
| 300 | chunks.push(TextChunk { |
| 301 | index, |
| 302 | start: seg_offset + sub.start, |
| 303 | end: seg_offset + sub.end, |
| 304 | text: sub.text, |
| 305 | }); |
| 306 | index += 1; |
| 307 | } |
| 308 | continue; |
| 309 | } |
| 310 | |
| 311 | let current_chars = current_text.chars().count(); |
| 312 | // Would adding this segment exceed chunk_size? |
| 313 | if current_chars + seg_chars > chunk_size && !current_text.is_empty() { |
| 314 | // Emit current chunk. |
| 315 | let start = current_start.unwrap_or(0); |
| 316 | let end = start + current_chars; |
| 317 | chunks.push(TextChunk { |
| 318 | index, |
| 319 | start, |
| 320 | end, |
| 321 | text: current_text.clone(), |
| 322 | }); |
| 323 | index += 1; |
| 324 | |
| 325 | // Apply overlap: keep the last `overlap` characters. |