Function build_hubert_graph

src/framework/modules/hubert_encoder.cpp:248–309

    const core::TensorValue & input_values,
    const HubertEncoderWeights & weights) {
    const auto & config = weights.config;
    auto hidden = core::reshape_tensor(
        ctx,
        input_values,
        core::TensorShape::from_dims({input_values.shape.dims[0], 1, input_values.shape.dims[1]}));
    int64_t in_channels = 1;
    int64_t frames = input_values.shape.dims[1];
    for (size_t index = 0; index < config.conv_dim.size(); ++index) {
        const std::string prefix = "feature_extractor.conv_layers." + std::to_string(index);
        hidden = Conv1dModule({
            in_channels,
            config.conv_dim[index],
            config.conv_kernel[index],
            static_cast<int>(config.conv_stride[index]),
            0,
            1,
            true}).build(ctx, hidden, conv1d_weights(weights, prefix + ".conv"));
        hidden = transpose_bct_btc(ctx, hidden);
        hidden = LayerNormModule({config.conv_dim[index], config.layer_norm_eps, true, true})
                     .build(ctx, hidden, norm_weights(weights, prefix + ".layer_norm"));
        hidden = transpose_bct_btc(ctx, hidden);
        hidden = GeluModule({GeluApproximation::ExactErf}).build(ctx, hidden);
        frames = conv1d_output_frames(frames, config.conv_kernel[index], config.conv_stride[index], 0);
        in_channels = config.conv_dim[index];
    }
    hidden = transpose_bct_btc(ctx, hidden);
    hidden = LayerNormModule({config.conv_dim.back(), config.layer_norm_eps, true, true})
                 .build(ctx, hidden, norm_weights(weights, "feature_projection.layer_norm"));
    hidden = LinearModule({config.conv_dim.back(), config.hidden_size, true, GGML_PREC_F32})
                 .build(ctx, hidden, linear_weights(weights, "feature_projection.projection"));

    if (config.apply_positional_embedding) {
        auto pos = grouped_pos_conv(
            ctx,
            transpose_bct_btc(ctx, hidden),
            conv1d_weights(weights, "encoder.pos_conv_embed.conv"),
            config);
        pos = transpose_bct_btc(ctx, pos);
        hidden = add_same(ctx, hidden, pos);
    }

    for (int64_t layer = 0; layer < config.output_hidden_layer; ++layer) {
        const std::string prefix = "encoder.layers." + std::to_string(layer);
        const auto attn_residual = hidden;
        hidden = LayerNormModule({config.hidden_size, config.layer_norm_eps, true, true})
                     .build(ctx, hidden, norm_weights(weights, prefix + ".layer_norm"));
        hidden = build_self_attention(ctx, hidden, weights, layer);
        hidden = add_same(ctx, attn_residual, hidden);
        const auto ff_in = LayerNormModule({config.hidden_size, config.layer_norm_eps, true, true})
                               .build(ctx, hidden, norm_weights(weights, prefix + ".final_layer_norm"));
        hidden = add_same(ctx, hidden, build_feed_forward(ctx, ff_in, weights, layer));
    }
    if (config.apply_final_layer_norm) {
        hidden = LayerNormModule({config.hidden_size, config.layer_norm_eps, true, true})
                     .build(ctx, hidden, norm_weights(weights, "encoder.layer_norm"));
    }
    return hidden;
}