Function _load_vllm_engine

funasr/bin/_server_app.py:53–91

Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable.

()

    }

    def _load_vllm_engine():
        """Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""
        if app.state.engine is not None:
            return
        try:
            from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
            from funasr import AutoModel as _AutoModel

            logger.info("Loading Fun-ASR-Nano vLLM engine...")
            t0 = time.time()
            app.state.engine = FunASRNanoVLLM.from_pretrained(
                model="FunAudioLLM/Fun-ASR-Nano-2512",
                hub="hf",
                device=device,
                dtype="bf16",
                max_model_len=4096,
                gpu_memory_utilization=0.5,
            )
            logger.info(f"vLLM engine ready in {time.time()-t0:.1f}s")
            app.state.use_vllm = True

            logger.info("Loading VAD model...")
            app.state.vad_model = _AutoModel(model="fsmn-vad", device=device, disable_update=True)
            logger.info("VAD ready.")
        except Exception as e:
            logger.warning(f"vLLM failed ({e}), falling back to AutoModel for fun-asr-nano")
            app.state.use_vllm = False
            from funasr import AutoModel
            cfg = {
                "model": "FunAudioLLM/Fun-ASR-Nano-2512",
                "hub": "hf",
                "trust_remote_code": True,
                "vad_model": "fsmn-vad",
                "vad_kwargs": {"max_single_segment_time": 30000},
                "device": device,
                "disable_update": True,
            }
            app.state.fallback_models["fun-asr-nano"] = AutoModel(**cfg)
            logger.info("Fallback AutoModel loaded for fun-asr-nano.")

    def _load_fallback(name: str):
        """Load non-LLM model via AutoModel."""