| 822 | } |
| 823 | |
| 824 | void run_step_into(int32_t token, SamplingLogits & logits) { |
| 825 | const auto & config = runtime_->assets().config; |
| 826 | if (step_cache_.valid_steps() >= cache_steps_) { |
| 827 | throw std::runtime_error("MioTTS lm decode cache exhausted"); |
| 828 | } |
| 829 | auto timing_start = Clock::now(); |
| 830 | ggml_backend_tensor_set(token_id_, &token, 0, sizeof(int32_t)); |
| 831 | const int32_t position = static_cast<int32_t>(step_cache_.current_end()); |
| 832 | ggml_backend_tensor_set(positions_, &position, 0, sizeof(int32_t)); |
| 833 | const int32_t cache_slot = static_cast<int32_t>(step_cache_.valid_steps()); |
| 834 | ggml_backend_tensor_set(cache_slot_, &cache_slot, 0, sizeof(int32_t)); |
| 835 | input_upload_ms_ += engine::debug::elapsed_ms(timing_start, Clock::now()); |
| 836 | const auto masked = ggml_fp32_to_fp16(-INFINITY); |
| 837 | const auto visible = ggml_fp32_to_fp16(0.0F); |
| 838 | std::fill(attention_mask_values_.begin(), attention_mask_values_.end(), masked); |
| 839 | for (int64_t i = 0; i < step_cache_.valid_steps(); ++i) { |
| 840 | attention_mask_values_[static_cast<size_t>(i)] = visible; |
| 841 | } |
| 842 | attention_mask_values_[static_cast<size_t>(cache_slot)] = visible; |
| 843 | timing_start = Clock::now(); |
| 844 | ggml_backend_tensor_set( |
| 845 | attention_mask_, |
| 846 | attention_mask_values_.data(), |
| 847 | 0, |
| 848 | attention_mask_values_.size() * sizeof(ggml_fp16_t)); |
| 849 | mask_upload_ms_ += engine::debug::elapsed_ms(timing_start, Clock::now()); |
| 850 | core::set_backend_threads(runtime_->backend(), runtime_->threads()); |
| 851 | timing_start = Clock::now(); |
| 852 | const ggml_status status = engine::core::compute_backend_graph(runtime_->backend(), graph_); |
| 853 | ggml_backend_synchronize(runtime_->backend()); |
| 854 | graph_compute_ms_ += engine::debug::elapsed_ms(timing_start, Clock::now()); |
| 855 | if (status != GGML_STATUS_SUCCESS) { |
| 856 | throw std::runtime_error("MioTTS lm decode graph compute failed"); |
| 857 | } |
| 858 | logits.speech.resize(static_cast<size_t>(config.speech_token_count)); |
| 859 | timing_start = Clock::now(); |
| 860 | ggml_backend_tensor_get( |
| 861 | logits_, |
| 862 | logits.speech.data(), |
| 863 | 0, |
| 864 | logits.speech.size() * sizeof(float)); |
| 865 | ggml_backend_tensor_get( |
| 866 | logits_, |
| 867 | &logits.eos, |
| 868 | static_cast<size_t>(config.speech_token_count) * sizeof(float), |
| 869 | sizeof(float)); |
| 870 | logits_read_ms_ += engine::debug::elapsed_ms(timing_start, Clock::now()); |
| 871 | step_cache_.advance_after_direct_append(1); |
| 872 | } |
| 873 | |
| 874 | void log_timing() const { |
| 875 | debug::timing_log_scalar("miotts.lm.decode.input_upload_ms", input_upload_ms_); |
no test coverage detected