Method api_server

lmdeploy/cli/serve.py:210–347

Serve LLMs with restful api using fastapi.

(args)

    @staticmethod
    def api_server(args):
        """Serve LLMs with restful api using fastapi."""
        from lmdeploy.archs import autoget_backend

        max_batch_size = args.max_batch_size if args.max_batch_size \
            else get_max_batch_size(args.device)
        backend = args.backend
        if backend != 'pytorch':
            # set auto backend mode
            backend = autoget_backend(args.model_path, trust_remote_code=args.trust_remote_code)

        if backend == 'pytorch':
            from lmdeploy.messages import PytorchEngineConfig
            adapters = get_lora_adapters(args.adapters)
            backend_config = PytorchEngineConfig(
                dtype=args.dtype,
                tp=args.tp,
                dp=args.dp,
                ep=args.ep,
                max_batch_size=max_batch_size,
                cache_max_entry_count=args.cache_max_entry_count,
                block_size=args.cache_block_seq_len,
                kernel_block_size=args.kernel_block_size,
                session_len=args.session_len,
                adapters=adapters,
                enable_prefix_caching=args.enable_prefix_caching,
                device_type=args.device,
                quant_policy=args.quant_policy,
                eager_mode=args.eager_mode,
                max_prefill_token_num=args.max_prefill_token_num,
                enable_microbatch=args.enable_microbatch,
                enable_eplb=args.enable_eplb,
                enable_metrics=not args.disable_metrics,
                role=EngineRole[args.role],
                migration_backend=MigrationBackend[args.migration_backend],
                model_format=args.model_format,
                hf_overrides=args.hf_overrides,
                disable_vision_encoder=args.disable_vision_encoder,
                logprobs_mode=args.logprobs_mode,
                dllm_block_length=args.dllm_block_length,
                dllm_unmasking_strategy=args.dllm_unmasking_strategy,
                dllm_denoising_steps=args.dllm_denoising_steps,
                dllm_confidence_threshold=args.dllm_confidence_threshold,
                enable_return_routed_experts=args.enable_return_routed_experts,
                distributed_executor_backend=args.distributed_executor_backend,
            )
        else:
            from lmdeploy.messages import TurbomindEngineConfig
            backend_config = TurbomindEngineConfig(dtype=args.dtype,
                                                   tp=args.tp,
                                                   dp=args.dp,
                                                   cp=args.cp,
                                                   nnodes=args.nnodes,
                                                   node_rank=args.node_rank,
                                                   dist_init_addr=args.dist_init_addr,
                                                   max_batch_size=max_batch_size,
                                                   session_len=args.session_len,
                                                   model_format=args.model_format,