Function sam3_encode_memory

sam3.cpp:11510–11715

}

static bool sam3_encode_memory(
    sam3_tracker& tracker, sam3_state& state, const sam3_model& model,
    int inst_id, const float* mask_logits, int mask_h, int mask_w,
    int frame_idx, bool is_cond, float obj_score) {
    const auto& hp = model.hparams;
    const int D = hp.neck_dim, MD = hp.mem_out_dim;
    const int H = sam3_eff_feat_size(state, hp);
    const int HIGH_RES = sam3_eff_img_size(state, hp);
    const int INTERPOL = H * 16;

    // Mask preprocessing: mask_logits → HIGH_RES → sigmoid → scale/bias → INTERPOL
    auto m_hires = sam3_bilinear_interpolate(mask_logits, mask_w, mask_h, HIGH_RES, HIGH_RES);
    const float sig_scale = hp.sigmoid_scale(), sig_bias = hp.sigmoid_bias();
    for (auto& v : m_hires) { float s = 1.0f / (1.0f + expf(-v)); v = s * sig_scale + sig_bias; }
    auto m_interp = sam3_bilinear_interpolate(m_hires.data(), HIGH_RES, HIGH_RES, INTERPOL, INTERPOL);

    const size_t bs = ggml_tensor_overhead() * 16384 + ggml_graph_overhead();
    struct ggml_init_params gp = {bs, nullptr, true};
    auto* ctx0 = ggml_init(gp);
    if (!ctx0) return false;

    auto* mask_in = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, INTERPOL, INTERPOL, 1, 1);
    ggml_set_name(mask_in, "mem_mask");
    ggml_set_input(mask_in);

    // Learned mask downsampler: 4 stages conv+LN+GELU + final 1×1
    // Conv output is in ggml "conv" layout. permute(1,2,0,3) converts to "internal" layout
    // where ne[0]=channel for LN2d. permute(2,0,1,3) converts back to "conv" layout.
    auto* ds = mask_in;
    for (int s = 0; s < 4; ++s) {
        int out_ch = (int)model.mem_enc.ds_conv_w[s]->ne[3];
        ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[s], ds, 2, 2, 1, 1, 1, 1);
        ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[s], 1, 1, out_ch, 1));
        ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3));  // conv→internal (ne[0]=channel)
        ds = sam3_layer_norm_2d(ctx0, ds, model.mem_enc.ds_norm_w[s], model.mem_enc.ds_norm_b[s]);
        ds = ggml_gelu(ctx0, ds);
        ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 2, 0, 1, 3));  // internal→conv
    }
    ds = ggml_conv_2d(ctx0, model.mem_enc.ds_conv_w[4], ds, 1, 1, 0, 0, 1, 1);
    ds = ggml_add(ctx0, ds, ggml_reshape_4d(ctx0, model.mem_enc.ds_conv_b[4], 1, 1, D, 1));
    ds = ggml_cont(ctx0, ggml_permute(ctx0, ds, 1, 2, 0, 3));  // conv→internal [D, ...]

    // Pixel projection — use fresh input tensor to avoid pulling in the
    // entire ViT+neck recomputation from state.neck_trk[2]'s dependency tree
    auto* pix_in_raw = ggml_new_tensor_4d(ctx0, GGML_TYPE_F32, D, H, H, 1);
    ggml_set_name(pix_in_raw, "mem_pix_feat");
    ggml_set_input(pix_in_raw);
    auto* pix_in = ggml_cont(ctx0, ggml_permute(ctx0, pix_in_raw, 2, 0, 1, 3));
    auto* pix = ggml_conv_2d(ctx0, model.mem_enc.pix_proj_w, pix_in, 1, 1, 0, 0, 1, 1);
    pix = ggml_add(ctx0, pix, ggml_reshape_4d(ctx0, model.mem_enc.pix_proj_b, 1, 1, D, 1));
    pix = ggml_cont(ctx0, ggml_permute(ctx0, pix, 1, 2, 0, 3));

    // Fusion: ADD (not multiply)
    auto* fused = ggml_add(ctx0, pix, ds);
    for (int i = 0; i < 2; ++i)
        fused = sam3_cxblock_forward(ctx0, fused,
                                     model.mem_enc.fuser_dw_w[i], model.mem_enc.fuser_dw_b[i],
                                     model.mem_enc.fuser_norm_w[i], model.mem_enc.fuser_norm_b[i],