Function sam3_precompute_geom_input

sam3.cpp:7803–8024

Pre-compute geometry encoder input on CPU: box embeddings + CLS token. Reads model weights from GPU, computes embeddings, returns float vector. Layout: [D, N_geo] row-major where N_geo = n_boxes + 1 (CLS last).

// Reads model weights from GPU, computes embeddings, returns float vector.
// Layout: [D, N_geo] row-major where N_geo = n_boxes + 1 (CLS last).
static std::vector<float> sam3_precompute_geom_input(
    const sam3_model& model,
    const sam3_pcs_params& params,
    const float* img_feats_data,  // [D, W, H] ggml-layout backbone features (nullable if no boxes)
    int W_feat, int H_feat) {
    const auto& ge = model.geom_enc;
    const int D = model.hparams.neck_dim;  // 256
    const int roi_size = 7;
    const int num_pos_feats = D / 2;  // 128

    // Collect all exemplar boxes
    struct box_info {
        float cx, cy, w, h;
        int label;
    };
    std::vector<box_info> boxes;
    for (const auto& b : params.pos_exemplars) {
        // API provides XYXY in original image space — convert to normalized CxCyWH [0,1]
        float cx = (b.x0 + b.x1) * 0.5f;
        float cy = (b.y0 + b.y1) * 0.5f;
        float bw = b.x1 - b.x0;
        float bh = b.y1 - b.y0;
        boxes.push_back({cx, cy, bw, bh, 0});  // label 0 = positive
    }
    for (const auto& b : params.neg_exemplars) {
        float cx = (b.x0 + b.x1) * 0.5f;
        float cy = (b.y0 + b.y1) * 0.5f;
        float bw = b.x1 - b.x0;
        float bh = b.y1 - b.y0;
        boxes.push_back({cx, cy, bw, bh, 1});  // label 1 = negative
    }

    const int n_boxes = (int)boxes.size();
    const int N_geo = n_boxes + 1;

    // Read needed weights from GPU
    std::vector<float> box_proj_w_data(4 * D), box_proj_b_data(D);
    std::vector<float> type_embed_data(D * 2);
    std::vector<float> cls_data(D);
    std::vector<float> box_pos_proj_w_data(258 * D), box_pos_proj_b_data(D);
    std::vector<float> box_pool_proj_w_data(7 * 7 * D * D), box_pool_proj_b_data(D);
    std::vector<float> img_pre_norm_w_data(D), img_pre_norm_b_data(D);

    auto read_f32 = [](struct ggml_tensor* t, float* dst, size_t n) {
        sam3_read_f32(t, dst, n);
    };

    read_f32(ge.box_proj_w, box_proj_w_data.data(), 4 * D);
    read_f32(ge.box_proj_b, box_proj_b_data.data(), D);
    read_f32(ge.type_embed, type_embed_data.data(), D * 2);
    read_f32(ge.cls_token, cls_data.data(), D);
    read_f32(ge.box_pos_proj_w, box_pos_proj_w_data.data(), 258 * D);
    read_f32(ge.box_pos_proj_b, box_pos_proj_b_data.data(), D);

    if (n_boxes > 0) {
        read_f32(ge.box_pool_proj_w, box_pool_proj_w_data.data(), 7 * 7 * D * D);
        read_f32(ge.box_pool_proj_b, box_pool_proj_b_data.data(), D);
        read_f32(ge.img_pre_norm_w, img_pre_norm_w_data.data(), D);