Method synthesize_voice_clone_impl

src/models/chatterbox/tts.cpp:144–314

}

ChatterboxVoiceCloneOutputs ChatterboxTtsComponent::synthesize_voice_clone_impl(
    const std::string & text,
    const ChatterboxConditionalsOutputs & conds,
    const ChatterboxVoiceCloneConfig & config) const {
    ChatterboxVoiceCloneOutputs outputs;
    outputs.normalized_text = normalize_chatterbox_tts_text(text);
    const std::string language = normalize_chatterbox_language_code(config.language);
    outputs.text_tokens = chatterbox_language_uses_multilingual_t3(language)
        ? encode_chatterbox_multilingual_text(*tokenizer_, outputs.normalized_text, language)
        : encode_chatterbox_english_text(*tokenizer_, outputs.normalized_text);
    outputs.text_tokens.insert(outputs.text_tokens.begin(), chatterbox_english_start_token_id(*tokenizer_));
    outputs.text_tokens.push_back(chatterbox_english_stop_token_id(*tokenizer_));

    outputs.prompt_prep_gen_ms = conds.gen_ms;
    outputs.prompt_prep_gen_prompt_mel_ms = conds.gen.prompt_mel_ms;
    outputs.prompt_prep_gen_speaker_ms = conds.gen.speaker_ms;
    outputs.prompt_prep_gen_tokenizer_ms = conds.gen.tokenizer_ms;
    outputs.prompt_prep_voice_encoder_ms = conds.voice_encoder_ms;
    outputs.prompt_prep_tokenizer_ms = conds.tokenizer_ms;

    T3GenerateRequest request;
    request.speaker_embedding = conds.t3.speaker_embedding;
    request.cond_prompt_speech_tokens = conds.t3.cond_prompt_speech_tokens;
    request.emotion_adv = conds.t3.emotion_adv;
    request.text_tokens = outputs.text_tokens;
    request.max_new_tokens = config.max_new_tokens;
    request.stop_on_eos = config.stop_on_eos;
    request.do_sample = config.do_sample;
    request.temperature = config.temperature;
    request.top_p = config.top_p;
    request.min_p = config.min_p;
    request.repetition_penalty = config.repetition_penalty;
    request.guidance_scale = config.guidance_scale;
    request.seed = config.seed;

    const auto t3_memory_before = capture_backend_memory_snapshot(execution_context_);
    const auto t3_started = std::chrono::steady_clock::now();
    const auto t3_outputs = t3_.generate_speech_tokens(request);
    outputs.t3_ms =
        engine::debug::elapsed_ms(t3_started);
    const auto t3_memory_after = capture_backend_memory_snapshot(execution_context_);
    if (t3_memory_after.available) {
        outputs.cuda_memory_total_bytes = t3_memory_after.total_bytes;
    } else if (t3_memory_before.available) {
        outputs.cuda_memory_total_bytes = t3_memory_before.total_bytes;
    }
    outputs.t3_prefix_cache_build_ms = t3_outputs.prefix_cache_build_ms;
    outputs.t3_decoder_cache_clone_ms = t3_outputs.decoder_cache_clone_ms;
    outputs.t3_prefill_runner_ms = t3_outputs.prefill_runner_ms;
    outputs.t3_decode_runner_ms = t3_outputs.decode_runner_ms;
    outputs.t3_logits_ms = t3_outputs.logits_ms;
    outputs.t3_sampling_ms = t3_outputs.sampling_ms;
    outputs.t3_next_embed_ms = t3_outputs.next_embed_ms;
    outputs.t3_cuda_memory_used_before_bytes = t3_memory_before.used_bytes;
    outputs.t3_cuda_memory_used_after_bytes = t3_memory_after.used_bytes;
    engine::debug::timing_log_scalar("chatterbox.voice_clone.t3.total_ms", outputs.t3_ms);
    engine::debug::timing_log_scalar("chatterbox.voice_clone.t3.prefix_cache.build_ms", outputs.t3_prefix_cache_build_ms);
    engine::debug::timing_log_scalar("chatterbox.voice_clone.t3.decoder_cache.clone_ms", outputs.t3_decoder_cache_clone_ms);