Method llm_build_plamo2

subprojects/llama.cpp/src/models/plamo2.cpp:3–93

#include "models.h"

llm_build_plamo2::llm_build_plamo2(const llama_model & model, const llm_graph_params & params) :
    llm_graph_context_mamba(params) {
    ggml_tensor * cur;
    ggml_tensor * inpL;

    // {n_embd, n_tokens}
    inpL = build_inp_embd(model.tok_embd);
    cb(inpL, "embedding_output", -1);

    ggml_tensor * inp_pos = build_inp_pos();

    auto * inp_hybrid = build_inp_mem_hybrid();

    ggml_tensor * inp_out_ids = build_inp_out_ids();

    for (int il = 0; il < n_layer; ++il) {
        ggml_tensor * residual = inpL;

        // ggml_graph_add_node(gf, model.layers[il].attn_norm);
        // cb(model.layers[il].attn_norm, "attn_norm", il);

        // pre_mixer_norm
        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);

        // check if this layer is Mamba or Attention
        bool is_mamba_layer = hparams.is_recurrent(il);

        if (is_mamba_layer) {
            // PLaMo-2 Mamba layer
            cur = build_plamo2_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il);
        } else {
            // PLaMo-2 Attention layer
            cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il);
        }

        // post_mixer_norm
        cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
        cb(cur, "attn_post_norm", il);

        // residual connection
        cur = ggml_add(ctx0, cur, residual);
        cb(cur, "attn_residual", il);
        residual = cur;

        // pre-ffn norm
        cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
        cb(cur, "ffn_pre_norm", il);

        // feed-forward network
        cur = build_ffn(cur,
                model.layers[il].ffn_up, NULL, NULL,
                NULL, NULL, NULL,
                model.layers[il].ffn_down, NULL, NULL,
                NULL, LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);
        cb(cur, "ffn_out", il);

        // post ffn norm
        cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);