Method chat

lmdeploy/pipeline.py:181–242

Chat. Args: prompt: prompt string or a tuple of (prompt, image or [images]). session: the chat session. gen_config: an instance of GenerationConfig. Default to None. stream_response: whether to stream the response. adapter_name: ad

(self,
             prompt: str | tuple[str, Image | list[Image]],
             session=None,
             gen_config: GenerationConfig | None = None,
             stream_response=False,
             adapter_name=None,
             **kwargs)

        self.async_engine.close()

    def chat(self,
             prompt: str | tuple[str, Image | list[Image]],
             session=None,
             gen_config: GenerationConfig | None = None,
             stream_response=False,
             adapter_name=None,
             **kwargs) -> Session | Iterator:
        """Chat.

        Args:
            prompt: prompt string or a tuple of (prompt, image or [images]).
            session: the chat session.
            gen_config: an instance of GenerationConfig. Default to None.
            stream_response: whether to stream the response.
            adapter_name: adapter name.
            **kwargs: additional keyword arguments.

        Returns:
            Session | Iterator: the updated session, or a streaming iterator if stream_response is True.
        """
        if session is None:
            session = self.session_mgr.get()
        session.update(prompt=prompt, response=None)

        prompt = MultimodalProcessor.format_prompts(prompt)

        sequence_start = session.step == 0
        generator = self.stream_infer(prompts=prompt,
                                      sessions=session,
                                      gen_config=gen_config,
                                      stream_response=stream_response,
                                      adapter_name=adapter_name,
                                      multiplex=True,
                                      sequence_start=sequence_start,
                                      sequence_end=False,
                                      step=session.step,
                                      **kwargs)

        def _gen():
            resp = None
            try:
                for out in generator:
                    resp = resp.extend(out) if resp else out
                    yield out
            except:  # noqa
                self._run(coro=session.async_abort())
                raise
            else:
                session.response = resp
                session.step += resp.generate_token_len + resp.input_token_len
                session.history.append((session.prompt, resp.text))

        if stream_response:
            return _gen()
        else:
            # run the generator until finish
            with closing(_gen()) as gen:
                for _ in gen: