Method _process_batch

optillm/batching.py:185–219

Process a batch of requests Args: batch: List of batch requests to process

(self, batch: List[BatchRequest])

        self.batch_threads[queue_key] = thread
    
    def _process_batch(self, batch: List[BatchRequest]) -> None:
        """
        Process a batch of requests
        
        Args:
            batch: List of batch requests to process
        """
        try:
            # Validate batch compatibility
            self._validate_batch_compatibility(batch)
            
            if not hasattr(self, '_processor_func'):
                raise BatchingError("No batch processor function set")
            
            # Extract request data
            request_data_list = [req.request_data for req in batch]
            
            # Process the batch
            responses = self._processor_func(request_data_list)
            
            # Validate response count
            if len(responses) != len(batch):
                raise BatchingError(f"Processor returned {len(responses)} responses for {len(batch)} requests")
            
            # Set results
            for req, response in zip(batch, responses):
                req.future.set_result(response)
                
        except Exception as e:
            # Fail all requests in batch with the same error
            error_msg = f"Batch processing failed: {str(e)}"
            logger.error(error_msg)
            
            for req in batch:
                req.future.set_exception(BatchingError(error_msg))
    
    def set_processor(self, processor_func):
        """