Function layer

src/framework/modules/text_encoders/t5_gemma_encoder.cpp:160–175

    const LinearModule up_proj({config.hidden_size, config.intermediate_size, false, GGML_PREC_F32});
    const LinearModule down_proj({config.intermediate_size, config.hidden_size, false, GGML_PREC_F32});
    auto gate = gate_proj.build(ctx, input, weights.gate_proj);
    gate = GeluModule({GeluApproximation::Tanh}).build(ctx, gate);
    auto up = up_proj.build(ctx, input, weights.up_proj);
    return down_proj.build(ctx, MulModule{}.build(ctx, gate, up), weights.down_proj);
}

core::TensorValue layer(
    core::ModuleBuildContext & ctx,
    const core::TensorValue & input,
    const core::TensorValue & positions,
    const core::TensorValue & additive_attention_mask,
    const T5GemmaEncoderLayerWeights & weights,
    const T5GemmaEncoderConfig & config) {
    auto hidden = gemma_rms_norm(ctx, input, weights.pre_self_attn_norm, config.rms_norm_eps, config.hidden_size);
    hidden = self_attention(ctx, hidden, positions, additive_attention_mask, weights, config);
    hidden = gemma_rms_norm(ctx, hidden, weights.post_self_attn_norm, config.rms_norm_eps, config.hidden_size);
    auto output = AddModule{}.build(ctx, input, hidden);
    hidden = gemma_rms_norm(ctx, output, weights.pre_ff_norm, config.rms_norm_eps, config.hidden_size);
    hidden = mlp(ctx, hidden, weights, config);