Method batch_iterator

rag_system/utils/batch_processor.py:130–133

Generate batches as an iterator for memory-efficient processing

(self, items: List[Any])

        return results
        
    def batch_iterator(self, items: List[Any]) -> Iterator[List[Any]]:
        """Generate batches as an iterator for memory-efficient processing"""
        for i in range(0, len(items), self.batch_size):
            yield items[i:i + self.batch_size]

class StreamingProcessor:
    """Process items one at a time with minimal memory usage"""