Estimate memory usage of chunks in MB
(chunks: List[Dict[str, Any]])
| 197 | return document_batches |
| 198 | |
| 199 | def estimate_memory_usage(chunks: List[Dict[str, Any]]) -> float: |
| 200 | """Estimate memory usage of chunks in MB""" |
| 201 | if not chunks: |
| 202 | return 0.0 |
| 203 | |
| 204 | # Rough estimate: average text length * number of chunks * 2 (for overhead) |
| 205 | avg_text_length = sum(len(chunk.get('text', '')) for chunk in chunks[:min(10, len(chunks))]) / min(10, len(chunks)) |
| 206 | estimated_bytes = avg_text_length * len(chunks) * 2 |
| 207 | return estimated_bytes / (1024 * 1024) # Convert to MB |
| 208 | |
| 209 | if __name__ == '__main__': |
| 210 | # Test the batch processor |
no outgoing calls
no test coverage detected