Manages Text-to-Speech (TTS) synthesis using various engines via RealtimeTTS. This class initializes a chosen TTS engine (Coqui, Kokoro, or Orpheus), configures it for streaming output, measures initial latency (TTFT), and provides methods to synthesize audio from text strings or g
| 66 | ) |
| 67 | |
| 68 | class AudioProcessor: |
| 69 | """ |
| 70 | Manages Text-to-Speech (TTS) synthesis using various engines via RealtimeTTS. |
| 71 | |
| 72 | This class initializes a chosen TTS engine (Coqui, Kokoro, or Orpheus), |
| 73 | configures it for streaming output, measures initial latency (TTFT), |
| 74 | and provides methods to synthesize audio from text strings or generators, |
| 75 | placing the resulting audio chunks into a queue. It handles dynamic |
| 76 | stream parameter adjustments and manages the synthesis lifecycle, including |
| 77 | optional callbacks upon receiving the first audio chunk. |
| 78 | """ |
| 79 | def __init__( |
| 80 | self, |
| 81 | engine: str = START_ENGINE, |
| 82 | orpheus_model: str = "orpheus-3b-0.1-ft-Q8_0-GGUF/orpheus-3b-0.1-ft-q8_0.gguf", |
| 83 | ) -> None: |
| 84 | """ |
| 85 | Initializes the AudioProcessor with a specific TTS engine. |
| 86 | |
| 87 | Sets up the chosen engine (Coqui, Kokoro, Orpheus), downloads Coqui models |
| 88 | if necessary, configures the RealtimeTTS stream, and performs an initial |
| 89 | synthesis to measure Time To First Audio chunk (TTFA). |
| 90 | |
| 91 | Args: |
| 92 | engine: The name of the TTS engine to use ("coqui", "kokoro", "orpheus"). |
| 93 | orpheus_model: The path or identifier for the Orpheus model file (used only if engine is "orpheus"). |
| 94 | """ |
| 95 | self.engine_name = engine |
| 96 | self.stop_event = threading.Event() |
| 97 | self.finished_event = threading.Event() |
| 98 | self.audio_chunks = asyncio.Queue() # Queue for synthesized audio output |
| 99 | self.orpheus_model = orpheus_model |
| 100 | |
| 101 | self.silence = ENGINE_SILENCES.get(engine, ENGINE_SILENCES[self.engine_name]) |
| 102 | self.current_stream_chunk_size = QUICK_ANSWER_STREAM_CHUNK_SIZE # Initial chunk size |
| 103 | |
| 104 | # Dynamically load and configure the selected TTS engine |
| 105 | if engine == "coqui": |
| 106 | ensure_lasinya_models(models_root="models", model_name="Lasinya") |
| 107 | self.engine = CoquiEngine( |
| 108 | specific_model="Lasinya", |
| 109 | local_models_path="./models", |
| 110 | voice="reference_audio.wav", |
| 111 | speed=1.1, |
| 112 | use_deepspeed=True, |
| 113 | thread_count=6, |
| 114 | stream_chunk_size=self.current_stream_chunk_size, |
| 115 | overlap_wav_len=1024, |
| 116 | load_balancing=True, |
| 117 | load_balancing_buffer_length=0.5, |
| 118 | load_balancing_cut_off=0.1, |
| 119 | add_sentence_filter=True, |
| 120 | ) |
| 121 | elif engine == "kokoro": |
| 122 | self.engine = KokoroEngine( |
| 123 | voice="af_heart", |
| 124 | default_speed=1.26, |
| 125 | trim_silence=True, |