Process audio with vLLM engine (Fun-ASR-Nano).
(audio_data, sr, language=None, hotwords=None, use_spk=False)
| 106 | return model |
| 107 | |
| 108 | def _process_vllm(audio_data, sr, language=None, hotwords=None, use_spk=False): |
| 109 | """Process audio with vLLM engine (Fun-ASR-Nano).""" |
| 110 | if sr != 16000: |
| 111 | import librosa |
| 112 | audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=16000) |
| 113 | sr = 16000 |
| 114 | if audio_data.ndim > 1: |
| 115 | audio_data = audio_data[:, 0] |
| 116 | audio_data = audio_data.astype(np.float32) |
| 117 | |
| 118 | # VAD |
| 119 | vad_res = app.state.vad_model.generate(input=audio_data, fs=sr) |
| 120 | segments = vad_res[0]["value"] if vad_res and vad_res[0].get("value") else [[0, int(len(audio_data)*1000/sr)]] |
| 121 | |
| 122 | seg_audios = [] |
| 123 | seg_times = [] |
| 124 | for seg in segments: |
| 125 | s0 = int(seg[0] * sr / 1000) |
| 126 | s1 = int(seg[1] * sr / 1000) |
| 127 | seg_audio = audio_data[s0:s1] |
| 128 | if len(seg_audio) > sr * 0.3: |
| 129 | seg_audios.append(seg_audio) |
| 130 | seg_times.append((seg[0], seg[1])) |
| 131 | |
| 132 | if not seg_audios: |
| 133 | return {"text": "", "segments": [], "duration": len(audio_data)/sr} |
| 134 | |
| 135 | # repetition_penalty is left at the neutral 1.0: the Fun-ASR-Nano vLLM |
| 136 | # engine runs in prompt-embeds mode, where any other value crashes the |
| 137 | # CUDA kernel (see issue #2948 and fun_asr_nano.vllm_utils). |
| 138 | gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.0} |
| 139 | if language: |
| 140 | gen_kwargs["language"] = language |
| 141 | if hotwords: |
| 142 | gen_kwargs["hotwords"] = hotwords |
| 143 | |
| 144 | results = app.state.engine.generate(inputs=seg_audios, **gen_kwargs) |
| 145 | |
| 146 | output_segments = [] |
| 147 | full_text_parts = [] |
| 148 | for r, (start_ms, end_ms) in zip(results, seg_times): |
| 149 | text = r["text"] |
| 150 | seg_info = {"text": text, "start": start_ms/1000, "end": end_ms/1000} |
| 151 | if "timestamps" in r: |
| 152 | offset = start_ms / 1000 |
| 153 | seg_info["words"] = [ |
| 154 | {"word": ts["token"], "start": ts["start_time"]+offset, "end": ts["end_time"]+offset} |
| 155 | for ts in r["timestamps"] |
| 156 | ] |
| 157 | output_segments.append(seg_info) |
| 158 | full_text_parts.append(text) |
| 159 | |
| 160 | return { |
| 161 | "text": "".join(full_text_parts), |
| 162 | "segments": output_segments, |
| 163 | "duration": len(audio_data) / sr, |
| 164 | } |
| 165 |
no test coverage detected
searching dependent graphs…