Method generate

swift/pipelines/eval/utils.py:104–147

Generate model response using batch inference. This method queues the request for batch processing and waits for the result. The actual inference is performed asynchronously in a background thread. Args: input: List of chat messages forming the conversa

(
        self,
        input: List[EvalChatMessage],
        tools: List[ToolInfo],
        tool_choice: ToolChoice,
        config: GenerateConfig,
    )

        self.engine = TransformersEngine(self.model, template=self.template, max_batch_size=self.max_batch_size)

    def generate(
        self,
        input: List[EvalChatMessage],
        tools: List[ToolInfo],
        tool_choice: ToolChoice,
        config: GenerateConfig,
    ) -> ModelOutput:
        """
        Generate model response using batch inference.

        This method queues the request for batch processing and waits for the result.
        The actual inference is performed asynchronously in a background thread.

        Args:
            input: List of chat messages forming the conversation
            tools: Available tools for function calling (if supported)
            tool_choice: Tool selection strategy
            config: Generation configuration

        Returns:
            ModelOutput containing the generated response
        """
        # Ensure the background batch processing thread is running
        global batch_thread
        if batch_thread is None:
            batch_thread = Thread(target=_process_batches, daemon=True)
            batch_thread.start()

        # Convert EvalScope format to ms-swift format
        ms_input = convert_request(input, tools)
        ms_config = convert_config(config)

        # Package the request for batch processing
        batch_input = BatchInferInput(
            ms_input=ms_input, ms_config=ms_config, batch_size=config.batch_size, engine=self.engine)

        # Create a future to receive the result asynchronously
        future = Future[ModelOutput]()

        # Queue the request for batch processing
        batch_queue.put(_QueueItem(input=batch_input, future=future))

        # Block until the result is available
        return future.result()


def _process_batches() -> None: