| 1773 | |
| 1774 | struct EncoderGraph { |
| 1775 | EncoderGraph( |
| 1776 | std::shared_ptr<const OmniVoiceAssets> assets, |
| 1777 | std::shared_ptr<const AudioTokenizerWeights> weights, |
| 1778 | core::ExecutionContext & execution_context, |
| 1779 | size_t graph_arena_bytes, |
| 1780 | int64_t acoustic_samples, |
| 1781 | int64_t semantic_samples, |
| 1782 | int64_t frames) |
| 1783 | : assets_(std::move(assets)), |
| 1784 | weights_(std::move(weights)), |
| 1785 | backend_(execution_context.backend()), |
| 1786 | backend_type_(execution_context.backend_type()), |
| 1787 | compute_threads_(std::max(1, execution_context.config().threads)), |
| 1788 | acoustic_sample_capacity_(acoustic_samples), |
| 1789 | semantic_sample_capacity_(semantic_samples), |
| 1790 | frame_capacity_(frames) { |
| 1791 | ggml_init_params params{graph_arena_bytes, nullptr, true}; |
| 1792 | ctx_.reset(ggml_init(params)); |
| 1793 | if (ctx_ == nullptr) { |
| 1794 | throw std::runtime_error("failed to initialize OmniVoice audio-tokenizer encoder graph context"); |
| 1795 | } |
| 1796 | core::ModuleBuildContext ctx{ctx_.get(), "omnivoice.audio_tokenizer.encode", backend_type_}; |
| 1797 | const auto & config = assets_->config.audio_tokenizer; |
| 1798 | const auto & semantic = config.semantic_model; |
| 1799 | semantic_downsample_factor_ = semantic_downsample_factor(config); |
| 1800 | |
| 1801 | acoustic_input_ = core::make_tensor( |
| 1802 | ctx, |
| 1803 | GGML_TYPE_F32, |
| 1804 | core::TensorShape::from_dims({1, 1, acoustic_sample_capacity_})); |
| 1805 | semantic_input_ = core::make_tensor( |
| 1806 | ctx, |
| 1807 | GGML_TYPE_F32, |
| 1808 | core::TensorShape::from_dims({1, 1, semantic_sample_capacity_})); |
| 1809 | downsample_indices_ = core::make_tensor(ctx, GGML_TYPE_I32, core::TensorShape::from_dims({frame_capacity_})); |
| 1810 | ggml_set_input(acoustic_input_.tensor); |
| 1811 | ggml_set_input(semantic_input_.tensor); |
| 1812 | ggml_set_input(downsample_indices_.tensor); |
| 1813 | |
| 1814 | auto hubert = semantic_input_; |
| 1815 | for (size_t i = 0; i < weights_->feature_extractor.convs.size(); ++i) { |
| 1816 | const auto & conv_weights = weights_->feature_extractor.convs[i]; |
| 1817 | hubert = build_conv1d_im2col_f32( |
| 1818 | ctx, |
| 1819 | hubert, |
| 1820 | conv_weights, |
| 1821 | static_cast<int>(semantic.conv_stride[i]), |
| 1822 | 0, |
| 1823 | 1); |
| 1824 | hubert = ensure_contiguous(ctx, hubert); |
| 1825 | if (i == 0) { |
| 1826 | hubert = group_norm_affine( |
| 1827 | ctx, |
| 1828 | hubert, |
| 1829 | semantic.conv_dim.front(), |
| 1830 | semantic.layer_norm_eps, |
| 1831 | weights_->feature_extractor.first_group_norm, |
| 1832 | &tensor_writer_); |
nothing calls this directly
no test coverage detected