Method build_attn_block

subprojects/llama.cpp/src/models/lfm2.cpp:76–114

}

ggml_tensor * llm_build_lfm2::build_attn_block(ggml_tensor *             cur,
                                               ggml_tensor *             inp_pos,
                                               llm_graph_input_attn_kv * inp_attn,
                                               int                       il) const {
    GGML_ASSERT(hparams.n_embd_v_gqa(il) == hparams.n_embd_k_gqa(il));
    const auto n_embd_head = hparams.n_embd_head_v;
    const auto n_head_kv   = hparams.n_head_kv(il);

    auto * q = build_lora_mm(model.layers[il].wq, cur);
    cb(q, "model.layers.{}.self_attn.q_proj", il);
    auto * k = build_lora_mm(model.layers[il].wk, cur);
    cb(k, "model.layers.{}.self_attn.k_proj", il);
    auto * v = build_lora_mm(model.layers[il].wv, cur);
    cb(v, "model.layers.{}.self_attn.v_proj", il);

    q = ggml_reshape_3d(ctx0, q, n_embd_head, n_head, n_tokens);
    k = ggml_reshape_3d(ctx0, k, n_embd_head, n_head_kv, n_tokens);
    v = ggml_reshape_3d(ctx0, v, n_embd_head, n_head_kv, n_tokens);

    // qk norm
    q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
    cb(q, "model.layers.{}.self_attn.q_layernorm", il);
    k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
    cb(k, "model.layers.{}.self_attn.k_layernorm", il);

    // RoPE
    q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
                      attn_factor, beta_fast, beta_slow);
    k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,
                      attn_factor, beta_fast, beta_slow);

    cur = build_attn(inp_attn,
            model.layers[il].wo, NULL,
            q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);

    cb(cur, "model.layers.{}.self_attn.out_proj", il);

    return cur;
}

ggml_tensor * llm_build_lfm2::build_shortconv_block(ggml_tensor * cur, llm_graph_input_rs * inp_recr, int il) {
    const auto *   mctx_cur     = static_cast<const llama_memory_hybrid_context *>(mctx)->get_recr();