Split text into overlapping chunks with stable IDs. Args: text: Raw document text chunk_size: Max words per chunk overlap: Word overlap between adjacent chunks Returns: List of dicts: {id, text, start_word, end_word}
(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP)
| 31 | |
| 32 | |
| 33 | def chunk_text(text: str, chunk_size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list: |
| 34 | """ |
| 35 | Split text into overlapping chunks with stable IDs. |
| 36 | |
| 37 | Args: |
| 38 | text: Raw document text |
| 39 | chunk_size: Max words per chunk |
| 40 | overlap: Word overlap between adjacent chunks |
| 41 | |
| 42 | Returns: |
| 43 | List of dicts: {id, text, start_word, end_word} |
| 44 | """ |
| 45 | words = text.split() |
| 46 | if not words: |
| 47 | return [] |
| 48 | |
| 49 | chunks = [] |
| 50 | start = 0 |
| 51 | while start < len(words): |
| 52 | end = min(start + chunk_size, len(words)) |
| 53 | chunk = " ".join(words[start:end]) |
| 54 | # Stable ID: MD5 of first 100 chars — deterministic across re-index runs |
| 55 | chunk_id = hashlib.md5(chunk[:100].encode("utf-8", errors="replace")).hexdigest()[:12] |
| 56 | chunks.append({ |
| 57 | "id": chunk_id, |
| 58 | "text": chunk, |
| 59 | "start_word": start, |
| 60 | "end_word": end, |
| 61 | }) |
| 62 | if end == len(words): |
| 63 | break |
| 64 | start += chunk_size - overlap |
| 65 | |
| 66 | return chunks |
| 67 | |
| 68 | |
| 69 | def index_file(filepath: str, category: str = "docs") -> list: |