Function process_batch_requests

optillm/server.py:1121–1246

Process a batch of requests using true batching when possible Args: batch_requests: List of request data dictionaries Returns: List of response dictionaries

(batch_requests)

        # Set up the batch processor function
        def process_batch_requests(batch_requests):
            """
            Process a batch of requests using true batching when possible

            Args:
                batch_requests: List of request data dictionaries

            Returns:
                List of response dictionaries
            """
            import time
            from optillm.batching import BatchingError

            if not batch_requests:
                return []

            logger.info(f"Processing batch of {len(batch_requests)} requests")

            # Check if we can use true batching (all requests compatible and using 'none' approach)
            can_use_true_batching = True
            first_req = batch_requests[0]

            # Check compatibility across all requests
            for req_data in batch_requests:
                if (req_data['stream'] or
                    req_data['approaches'] != first_req['approaches'] or
                    req_data['operation'] != first_req['operation'] or
                    req_data['model'] != first_req['model']):
                    can_use_true_batching = False
                    break

            # For now, implement sequential processing but with proper infrastructure
            # TODO: Implement true PyTorch/MLX batching in next phase
            responses = []

            for i, req_data in enumerate(batch_requests):
                try:
                    logger.debug(f"Processing batch request {i+1}/{len(batch_requests)}")

                    # Extract request parameters
                    system_prompt = req_data['system_prompt']
                    initial_query = req_data['initial_query']
                    client = req_data['client']
                    model = req_data['model']
                    request_config = req_data['request_config']
                    approaches = req_data['approaches']
                    operation = req_data['operation']
                    n = req_data['n']
                    stream = req_data['stream']

                    # Validate request
                    if stream:
                        raise BatchingError("Streaming requests cannot be batched")

                    # Check if any of the approaches is 'none'
                    contains_none = any(approach == 'none' for approach in approaches)

                    if operation == 'SINGLE' and approaches[0] == 'none':