(device: str = "cuda", preload_model: str = "auto")
| 27 | |
| 28 | |
| 29 | def create_app(device: str = "cuda", preload_model: str = "auto") -> FastAPI: |
| 30 | if preload_model == "auto": |
| 31 | preload_model = "fun-asr-nano" if device.startswith("cuda") else "sensevoice" |
| 32 | |
| 33 | app = FastAPI(title="FunASR Server", version="1.3.6") |
| 34 | app.state.device = device |
| 35 | app.state.engine = None |
| 36 | app.state.vad_model = None |
| 37 | app.state.fallback_models = {} |
| 38 | |
| 39 | # Non-LLM model configs (use AutoModel, no vLLM) |
| 40 | FALLBACK_CONFIGS = { |
| 41 | "sensevoice": { |
| 42 | "model": "iic/SenseVoiceSmall", |
| 43 | "vad_model": "fsmn-vad", |
| 44 | "vad_kwargs": {"max_single_segment_time": 30000}, |
| 45 | }, |
| 46 | "paraformer": { |
| 47 | "model": "paraformer-zh", |
| 48 | "vad_model": "fsmn-vad", |
| 49 | "punc_model": "ct-punc", |
| 50 | }, |
| 51 | } |
| 52 | |
| 53 | def _load_vllm_engine(): |
| 54 | """Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable.""" |
| 55 | if app.state.engine is not None: |
| 56 | return |
| 57 | try: |
| 58 | from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM |
| 59 | from funasr import AutoModel as _AutoModel |
| 60 | |
| 61 | logger.info("Loading Fun-ASR-Nano vLLM engine...") |
| 62 | t0 = time.time() |
| 63 | app.state.engine = FunASRNanoVLLM.from_pretrained( |
| 64 | model="FunAudioLLM/Fun-ASR-Nano-2512", |
| 65 | hub="hf", |
| 66 | device=device, |
| 67 | dtype="bf16", |
| 68 | max_model_len=4096, |
| 69 | gpu_memory_utilization=0.5, |
| 70 | ) |
| 71 | logger.info(f"vLLM engine ready in {time.time()-t0:.1f}s") |
| 72 | app.state.use_vllm = True |
| 73 | |
| 74 | logger.info("Loading VAD model...") |
| 75 | app.state.vad_model = _AutoModel(model="fsmn-vad", device=device, disable_update=True) |
| 76 | logger.info("VAD ready.") |
| 77 | except Exception as e: |
| 78 | logger.warning(f"vLLM failed ({e}), falling back to AutoModel for fun-asr-nano") |
| 79 | app.state.use_vllm = False |
| 80 | from funasr import AutoModel |
| 81 | cfg = { |
| 82 | "model": "FunAudioLLM/Fun-ASR-Nano-2512", |
| 83 | "hub": "hf", |
| 84 | "trust_remote_code": True, |
| 85 | "vad_model": "fsmn-vad", |
| 86 | "vad_kwargs": {"max_single_segment_time": 30000}, |
no test coverage detected
searching dependent graphs…