Method generate

fastdeploy/engine/async_llm.py:455–569

Async generation interface Args: prompt: Input prompt sampling_params: Sampling parameters. If `sampling_params.n > 1`, will generate `n` completions sequentially. request_id: Request ID **kwargs: Other parameters

(
        self,
        prompt: Union[str, List[str], Dict[str, Any]],
        sampling_params: Optional[SamplingParams] = None,
        request_id: Optional[str] = None,
        **kwargs,
    )

            raise EngineError(f"async_llm add request failed: {e}", error_code=400)

    async def generate(
        self,
        prompt: Union[str, List[str], Dict[str, Any]],
        sampling_params: Optional[SamplingParams] = None,
        request_id: Optional[str] = None,
        **kwargs,
    ) -> AsyncGenerator[RequestOutput, None]:
        """
        Async generation interface

        Args:
            prompt: Input prompt
            sampling_params: Sampling parameters. If `sampling_params.n > 1`,
                will generate `n` completions sequentially.
            request_id: Request ID
            **kwargs: Other parameters

        Yields:
            RequestOutput: Generated output
        """

        num_choices = sampling_params.n if sampling_params is not None and sampling_params.n else 1
        stream = True
        include_stop_str_in_output = False
        enable_thinking = kwargs.pop("enable_thinking", False)

        if isinstance(prompt, dict):
            num_choices = prompt.get("n")
            stream = prompt.get("stream", True)
            include_stop_str_in_output = prompt.get("include_stop_str_in_output", False)

        # Ensure ZMQ client and connection manager are initialized in current process
        if (
            self.request_client is None
            or self.connection_manager is None
            or not getattr(self.connection_manager, "running", False)
        ):
            raise EngineError(
                "AsyncLLM engine not initialized. Call init_connections() before generate.",
                error_code=500,
            )

        # Build request ids and connection key
        if num_choices <= 1:
            # Single-choice: keep user-provided request_id semantics
            child_request_ids = [request_id or str(uuid.uuid4())]
            conn_request_id = child_request_ids[0]
        else:
            # Multi-choice: use unified "cmpl-" base id so DealerConnectionManager
            # can merge cmpl-xxx_0, cmpl-xxx_1, ... back to the same response queue.
            user_request_id = request_id or str(uuid.uuid4())
            conn_request_id = f"cmpl-{user_request_id}"
            child_request_ids = [f"{conn_request_id}_{i}" for i in range(num_choices)]

        try:
            # 1) Send all sub-requests to engine
            for child_request_id in child_request_ids:
                await self.add_request(child_request_id, prompt, sampling_params, **kwargs)