Split text into chunks with optional overlap.
(self, text: str, max_tokens: int, overlap_tokens: int = 0)
| 296 | ) |
| 297 | |
| 298 | def _split_text_into_chunks(self, text: str, max_tokens: int, overlap_tokens: int = 0) -> List[str]: |
| 299 | """Split text into chunks with optional overlap.""" |
| 300 | if not text.strip(): |
| 301 | return [] |
| 302 | |
| 303 | tokens = self._tokenizer.encode(text) |
| 304 | if len(tokens) <= max_tokens: |
| 305 | return [text] |
| 306 | |
| 307 | chunks = [] |
| 308 | start = 0 |
| 309 | while start < len(tokens): |
| 310 | end = min(start + max_tokens, len(tokens)) |
| 311 | chunk_tokens = tokens[start:end] |
| 312 | chunks.append(self._tokenizer.decode(chunk_tokens)) |
| 313 | |
| 314 | if end >= len(tokens): |
| 315 | break |
| 316 | |
| 317 | # Move start with overlap |
| 318 | start = end - overlap_tokens if overlap_tokens > 0 else end |
| 319 | |
| 320 | return chunks |
| 321 | |
| 322 | def _truncate_to_tokens(self, text: str, max_tokens: int) -> str: |
| 323 | """Truncate text to max tokens.""" |