Method add_request

optillm/batching.py:230–272

Add a request to be batched Args: request_data: The request data dictionary Returns: The response from batch processing Raises: BatchingError: If request cannot be processed

(self, request_data: Dict[str, Any])

        self._processor_func = processor_func
    
    def add_request(self, request_data: Dict[str, Any]) -> Dict[str, Any]:
        """
        Add a request to be batched
        
        Args:
            request_data: The request data dictionary
            
        Returns:
            The response from batch processing
            
        Raises:
            BatchingError: If request cannot be processed
        """
        try:
            # Generate key for request grouping
            queue_key = self._get_request_key(request_data)
            
            # Create queue and processor if needed
            if queue_key not in self.queues:
                self.queues[queue_key] = queue.Queue()
                self._create_batch_processor(queue_key)
            
            # Create batch request
            future = Future()
            batch_request = BatchRequest(
                request_data=request_data,
                future=future,
                timestamp=time.time(),
                model=request_data.get('model', 'default'),
                approach=request_data.get('optillm_approach')
            )
            
            # Add to appropriate queue
            self.queues[queue_key].put(batch_request)
            
            if self.enable_logging:
                logger.debug(f"Added request to batch queue {queue_key}")
            
            # Wait for result
            return future.result()
            
        except Exception as e:
            raise BatchingError(f"Failed to process request: {str(e)}")
    
    def get_stats(self) -> Dict[str, Any]:
        """Get batching statistics"""