Method build

src/framework/modules/attention/transformer_blocks.cpp:24–49

}

core::TensorValue TransformerEncoderBlockModule::build(
    core::ModuleBuildContext & ctx,
    const core::TensorValue & input,
    const TransformerEncoderBlockWeights & weights) const {
    validate_sequence_input(input, config_.hidden_size, "input");

    const LayerNormModule norm1(make_norm_config(config_.hidden_size, config_.eps));
    const SelfAttentionModule self_attention({config_.hidden_size, config_.num_heads, config_.use_bias});
    const LayerNormModule norm2(make_norm_config(config_.hidden_size, config_.eps));
    const FeedForwardModule feed_forward(
        {config_.hidden_size, config_.intermediate_size, config_.use_bias, GeluApproximation::ExactErf});
    const ResidualAddModule add;

    auto cur = norm1.build(ctx, input, weights.norm1);
    cur = self_attention.build(ctx, cur, weights.self_attention);
    if (weights.layer_scale1.has_value()) {
        cur = LayerScaleModule().build(ctx, cur, *weights.layer_scale1);
    }
    cur = add.build(ctx, input, cur);
    auto ff_in = norm2.build(ctx, cur, weights.norm2);
    auto ff_out = feed_forward.build(ctx, ff_in, weights.feed_forward);
    if (weights.layer_scale2.has_value()) {
        ff_out = LayerScaleModule().build(ctx, ff_out, *weights.layer_scale2);
    }
    return add.build(ctx, cur, ff_out);
}

const core::ModuleSchema & TransformerEncoderBlockModule::static_schema() noexcept {
    return kTransformerEncoderBlockSchema;