Method chat

fastdeploy/entrypoints/llm.py:216–291

Args: messages (Union[list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]]): Single conversation or a list of conversations. sampling_params (Optional[Union[SamplingParams, list[SamplingParams]]], optional): The sa

(
        self,
        messages: Union[list[Any], list[list[Any]]],
        sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
        use_tqdm: bool = True,
        chat_template_kwargs: Optional[dict[str, Any]] = None,
        chat_template: Optional[str] = None,
        tools: Optional[Union[ChatCompletionToolsParam, list[ChatCompletionToolsParam]]] = None,
        stream: bool = False,
    )

            return outputs

    def chat(
        self,
        messages: Union[list[Any], list[list[Any]]],
        sampling_params: Optional[Union[SamplingParams, list[SamplingParams]]] = None,
        use_tqdm: bool = True,
        chat_template_kwargs: Optional[dict[str, Any]] = None,
        chat_template: Optional[str] = None,
        tools: Optional[Union[ChatCompletionToolsParam, list[ChatCompletionToolsParam]]] = None,
        stream: bool = False,
    ):
        """
        Args:
            messages (Union[list[ChatCompletionMessageParam], list[list[ChatCompletionMessageParam]]]):
                Single conversation or a list of conversations.
            sampling_params (Optional[Union[SamplingParams, list[SamplingParams]]], optional):
                The sampling parameters to use for generating the response. Defaults to None.
            use_tqdm (bool, optional): Whether to use tqdm for the progress bar. Defaults to True.
            chat_template_kwargs(Optional[dict[str,Any]]): Additional kwargs to pass to the chat
                template.
            stream (bool, optional): Whether to return a streaming iterator. Defaults to False.

        Returns:
            If stream=False: Union[str, list[str]]: The generated response.
            If stream=True: Iterator: An iterator that yields partial responses as they become available.
        """

        if not self._check_master():
            err_msg = f"Only master node can accept completion request, please send request to master node: {self.master_node_ip}"
            raise ValueError(err_msg)

        if sampling_params is None:
            sampling_params = self.default_sampling_params

        if isinstance(sampling_params, SamplingParams):
            sampling_params_len = 1
        else:
            sampling_params_len = len(sampling_params)

        if isinstance(messages, list) and isinstance(messages[0], dict):
            messages = [messages]

        if sampling_params_len != 1 and len(messages) != sampling_params_len:
            raise ValueError("messages and sampling_params must be the same length.")

        if chat_template is None:
            chat_template = self.chat_template

        validated_tools = None
        if tools is not None:
            try:
                validated_tools = self._validate_tools(tools)
            except ValueError as e:
                raise RuntimeError(f"Failed to validate 'tools' parameter in chat method: {e}") from e

        req_ids = self._add_request(
            prompts=[{"messages": msg} for msg in messages],
            sampling_params=sampling_params,
            chat_template_kwargs=chat_template_kwargs,