| 2138 | } |
| 2139 | |
| 2140 | runtime::AudioBuffer run(const OmniVoiceGeneratedAudioTokens & codes) { |
| 2141 | if (codes.frames > frame_capacity_ || codes.codebooks > codebook_capacity_) { |
| 2142 | throw std::runtime_error("OmniVoice audio-tokenizer decoder request exceeds prepared capacity"); |
| 2143 | } |
| 2144 | for (int64_t codebook = 0; codebook < codebook_capacity_; ++codebook) { |
| 2145 | auto & ids = code_input_host_[static_cast<size_t>(codebook)]; |
| 2146 | std::fill(ids.begin(), ids.end(), 0); |
| 2147 | for (int64_t frame = 0; frame < codes.frames; ++frame) { |
| 2148 | ids[static_cast<size_t>(frame)] = |
| 2149 | codebook < codes.codebooks |
| 2150 | ? codes.token_ids[static_cast<size_t>(frame * codes.codebooks + codebook)] |
| 2151 | : 0; |
| 2152 | } |
| 2153 | if (frame_capacity_ > 0) { |
| 2154 | ggml_backend_tensor_set( |
| 2155 | code_inputs_[static_cast<size_t>(codebook)], |
| 2156 | ids.data(), |
| 2157 | 0, |
| 2158 | static_cast<size_t>(frame_capacity_) * sizeof(int32_t)); |
| 2159 | } |
| 2160 | } |
| 2161 | std::fill(decoder_frame_mask_host_.begin(), decoder_frame_mask_host_.end(), 0.0F); |
| 2162 | std::fill_n(decoder_frame_mask_host_.begin(), static_cast<size_t>(codes.frames), 1.0F); |
| 2163 | if (frame_capacity_ > 0) { |
| 2164 | ggml_backend_tensor_set( |
| 2165 | decoder_frame_mask_.tensor, |
| 2166 | decoder_frame_mask_host_.data(), |
| 2167 | 0, |
| 2168 | static_cast<size_t>(frame_capacity_) * sizeof(float)); |
| 2169 | } |
| 2170 | int64_t current_length = codes.frames; |
| 2171 | for (size_t i = 0; i < decoder_block_masks_.size(); ++i) { |
| 2172 | current_length *= assets_->config.audio_tokenizer.acoustic_model.upsampling_ratios[i]; |
| 2173 | auto & block_mask = decoder_block_mask_host_[i]; |
| 2174 | std::fill(block_mask.begin(), block_mask.end(), 0.0F); |
| 2175 | std::fill_n(block_mask.begin(), static_cast<size_t>(current_length), 1.0F); |
| 2176 | ggml_backend_tensor_set( |
| 2177 | decoder_block_masks_[i].tensor, |
| 2178 | block_mask.data(), |
| 2179 | 0, |
| 2180 | static_cast<size_t>(decoder_block_masks_[i].shape.dims[2]) * sizeof(float)); |
| 2181 | } |
| 2182 | core::set_backend_threads(backend_, compute_threads_); |
| 2183 | const ggml_status status = engine::core::compute_backend_graph(backend_, graph_); |
| 2184 | ggml_backend_synchronize(backend_); |
| 2185 | if (status != GGML_STATUS_SUCCESS) { |
| 2186 | throw std::runtime_error("OmniVoice audio-tokenizer decoder graph compute failed"); |
| 2187 | } |
| 2188 | runtime::AudioBuffer audio = {}; |
| 2189 | audio.sample_rate = assets_->config.audio_tokenizer.sample_rate; |
| 2190 | audio.channels = 1; |
| 2191 | const int64_t actual_samples = codes.frames * assets_->config.audio_tokenizer.hop_length; |
| 2192 | audio.samples.resize(static_cast<size_t>(actual_samples)); |
| 2193 | if (actual_samples > 0) { |
| 2194 | ggml_backend_tensor_get( |
| 2195 | output_, |
| 2196 | audio.samples.data(), |
| 2197 | 0, |
nothing calls this directly
no test coverage detected