Method generate

lmdeploy/serve/core/async_engine.py:475–781

Generate responses. Args: messages (str | List): chat history or prompt session_id (int | Session): the session id or instance of Session gen_config (GenerationConfig | None): a instance of GenerationConfig. Default to None. st

(
            self,
            messages,
            session_id: int | Session,
            gen_config: GenerationConfig | None = None,
            tools: list[object] | None = None,
            reasoning_effort: Literal['low', 'medium', 'high'] | None = None,
            stream_response: bool = True,
            sequence_start: bool = True,
            sequence_end: bool = True,  # no interactive mode by default
            step: int = 0,
            do_preprocess: bool = True,
            adapter_name: str | None = None,
            rewind_stop_tokens: bool = False,
            input_ids: list | None = None,
            enable_thinking: bool | None = None,
            chat_template_kwargs: dict | None = None,
            media_io_kwargs: dict[str, Any] | None = None,
            mm_processor_kwargs: dict[str, Any] | None = None,
            **kwargs)

            metrics_processor.decrease_api_routed_requests()

    async def generate(
            self,
            messages,
            session_id: int | Session,
            gen_config: GenerationConfig | None = None,
            tools: list[object] | None = None,
            reasoning_effort: Literal['low', 'medium', 'high'] | None = None,
            stream_response: bool = True,
            sequence_start: bool = True,
            sequence_end: bool = True,  # no interactive mode by default
            step: int = 0,
            do_preprocess: bool = True,
            adapter_name: str | None = None,
            rewind_stop_tokens: bool = False,
            input_ids: list | None = None,
            enable_thinking: bool | None = None,
            chat_template_kwargs: dict | None = None,
            media_io_kwargs: dict[str, Any] | None = None,
            mm_processor_kwargs: dict[str, Any] | None = None,
            **kwargs):
        """Generate responses.

        Args:
            messages (str | List): chat history or prompt
            session_id (int | Session): the session id or instance of Session
            gen_config (GenerationConfig | None): a instance of
                GenerationConfig. Default to None.
            stream_response (bool): whether return responses streamingly
            sequence_start (bool): indicator for starting a sequence
            sequence_end (bool): indicator for ending a sequence
            step (int): the offset of the k/v cache
            do_preprocess (bool): whether pre-process the messages. Default to
                True, which means chat_template will be applied.
        """
        metrics_processor.increase_total_requests()

        if (messages is not None) ^ (input_ids is None):
            raise ValueError('You must specify exactly one of messages or input_ids')
        if isinstance(session_id, Session):
            session = session_id
        elif isinstance(session_id, int):
            session = self.session_mgr.get(session_id, step=step)
        else:
            raise ValueError(f'Invalid session_id: {session_id}. It should be an instance of Session or an integer.')
        session_id = session.session_id
        session_removed = False

        def remove_session_once():
            nonlocal session_removed
            if sequence_end and not session_removed:
                self.session_mgr.remove(session)
                session_removed = True

        chat_template_kwargs = chat_template_kwargs or {}
        if enable_thinking is not None:
            logger.warning('enable_thinking is deprecated, use chat_template_kwargs["enable_thinking"] instead')
            if chat_template_kwargs.get('enable_thinking') is None:
                chat_template_kwargs['enable_thinking'] = enable_thinking