Chunks the text property of an object into smaller pieces with overlap. Args: obj (dict): The input object containing properties including 'text' chunk_size (int): Maximum size of each chunk overlap (int): Number of characters to overlap between chunks
(obj, chunk_size=1000, overlap=200)
| 1 | def chunk_object(obj, chunk_size=1000, overlap=200): |
| 2 | """ |
| 3 | Chunks the text property of an object into smaller pieces with overlap. |
| 4 | |
| 5 | Args: |
| 6 | obj (dict): The input object containing properties including 'text' |
| 7 | chunk_size (int): Maximum size of each chunk |
| 8 | overlap (int): Number of characters to overlap between chunks |
| 9 | |
| 10 | Returns: |
| 11 | list: List of objects with the same properties but chunked text |
| 12 | """ |
| 13 | # If there's no text property or text is shorter than chunk_size, return original |
| 14 | if 'text' not in obj or len(obj['text']) <= chunk_size: |
| 15 | return [obj] |
| 16 | |
| 17 | text = obj['text'] |
| 18 | chunks = [] |
| 19 | start = 0 |
| 20 | |
| 21 | while start < len(text): |
| 22 | # Get chunk of text |
| 23 | end = start + chunk_size |
| 24 | |
| 25 | # If this isn't the first chunk, include the overlap from the previous chunk |
| 26 | if start > 0: |
| 27 | start = start - overlap |
| 28 | |
| 29 | # Get the chunk |
| 30 | chunk = text[start:end] |
| 31 | |
| 32 | # Create new object with same properties but chunked text |
| 33 | new_obj = obj.copy() |
| 34 | new_obj['text'] = chunk |
| 35 | |
| 36 | |
| 37 | chunks.append(new_obj) |
| 38 | |
| 39 | # Move start position for next chunk |
| 40 | start = end |
| 41 | |
| 42 | return chunks |