Function pipeline

lmdeploy/api.py:15–80

Create a pipeline for inference. Args: model_path: the path of a model. It could be one of the following options: - i) A local directory path of a turbomind model which is converted by ``lmdeploy convert`` command or download from ii) and iii).

(model_path: str,
             backend_config: TurbomindEngineConfig | PytorchEngineConfig | None = None,
             chat_template_config: ChatTemplateConfig | None = None,
             log_level: str = 'WARNING',
             max_log_len: int | None = None,
             trust_remote_code: bool = False,
             speculative_config: SpeculativeConfig | None = None,
             **kwargs)

def pipeline(model_path: str,
             backend_config: TurbomindEngineConfig | PytorchEngineConfig | None = None,
             chat_template_config: ChatTemplateConfig | None = None,
             log_level: str = 'WARNING',
             max_log_len: int | None = None,
             trust_remote_code: bool = False,
             speculative_config: SpeculativeConfig | None = None,
             **kwargs):
    """Create a pipeline for inference.

    Args:
        model_path: the path of a model. It could be one of the following options:

            - i) A local directory path of a turbomind model which is
              converted by ``lmdeploy convert`` command or download from
              ii) and iii).
            - ii) The model_id of a lmdeploy-quantized model hosted
              inside a model repo on huggingface.co, such as
              ``InternLM/internlm-chat-20b-4bit``,
              ``lmdeploy/llama2-chat-70b-4bit``, etc.
            - iii) The model_id of a model hosted inside a model repo
              on huggingface.co, such as ``internlm/internlm-chat-7b``,
              ``Qwen/Qwen-7B-Chat``, ``baichuan-inc/Baichuan2-7B-Chat``
              and so on.
        backend_config: backend config instance. Default to None.
        chat_template_config: chat template configuration. Default to None.
        log_level: set log level whose value among [``CRITICAL``, ``ERROR``,
            ``WARNING``, ``INFO``, ``DEBUG``]
        max_log_len: Max number of prompt characters or prompt tokens
            being printed in log.
        trust_remote_code: whether to trust remote code from model repositories.
        speculative_config: speculative decoding configuration.
        **kwargs: additional keyword arguments passed to the pipeline.

    Returns:
        Pipeline: a pipeline instance for inference.

    Examples:

        .. code-block:: python

            # LLM
            import lmdeploy
            pipe = lmdeploy.pipeline('internlm/internlm-chat-7b')
            response = pipe(['hi','say this is a test'])
            print(response)

            # VLM
            from lmdeploy.vl import load_image
            from lmdeploy import pipeline, TurbomindEngineConfig, ChatTemplateConfig
            pipe = pipeline('liuhaotian/llava-v1.5-7b',
                            backend_config=TurbomindEngineConfig(session_len=8192),
                            chat_template_config=ChatTemplateConfig(model_name='vicuna'))
            im = load_image('https://raw.githubusercontent.com/open-mmlab/mmdeploy/main/demo/resources/human-pose.jpg')
            response = pipe([('describe this image', [im])])
            print(response)
    """  # noqa E501