Function sam3_segment_pcs

sam3.cpp:9665–10162

*****************************************************************************/

sam3_result sam3_segment_pcs(sam3_state& state,
                             const sam3_model& model,
                             const sam3_pcs_params& params) {
    if (model.hparams.visual_only || model.hparams.is_sam2()) {
        fprintf(stderr, "%s: ERROR: PCS not available on %s model\n",
                __func__, model.hparams.is_sam2() ? "SAM2" : "visual-only");
        return sam3_result{};
    }

#if SAM3_LOG_LEVEL >= 1
    auto t_start = std::chrono::high_resolution_clock::now();
#endif
    const auto& hp = model.hparams;
    const int D = hp.neck_dim;           // 256
    const int H = hp.n_img_embd();       // 72
    const int L = hp.text_ctx_len;       // 32
    const int NQ = hp.ddec_num_queries;  // 200
    const int N_spatial = H * H;         // 5184
    sam3_result result;

    // ── Check that image has been encoded ────────────────────────────────
    if (!state.neck_det[0]) {
        fprintf(stderr, "%s: image not encoded — call sam3_encode_image first\n", __func__);
        return result;
    }

    // ── Tokenize text prompt ─────────────────────────────────────────────
    auto token_ids = sam3_tokenize(const_cast<sam3_bpe_tokenizer&>(model.tokenizer),
                                   params.text_prompt, L);
    if (token_ids.empty()) {
        fprintf(stderr, "%s: failed to tokenize text prompt\n", __func__);
        return result;
    }

    SAM3_LOG(2, "%s: text='%s', %zu tokens\n", __func__,
             params.text_prompt.c_str(), token_ids.size());

    // ── Helper: run a sub-graph with its own context and allocator ──────
    // Each stage below follows this exact pattern:
    //   1. Create fresh ggml_context + graph + allocator
    //   2. Create INPUT tensors (ggml_set_input)
    //   3. Build the computation graph
    //   4. Mark outputs (ggml_set_output)
    //   5. Allocate → set input data → compute → read output data
    //   6. Free allocator + context
    // This ensures ZERO buffer sharing between stages.

    // ── Pre-compute shared CPU data used by multiple stages ─────────────
    const int n_boxes = (int)(params.pos_exemplars.size() + params.neg_exemplars.size());
    const int N_geo = n_boxes + 1;  // +1 for CLS
    const int T = L + N_geo;        // total prompt tokens (text + geometry)

    // Read image features and PE from state into CPU buffers (used by multiple stages)
    std::vector<float> img_feats_cpu(D * N_spatial);
    std::vector<float> img_pe_cpu(D * N_spatial);
    ggml_backend_tensor_get(state.neck_det[2], img_feats_cpu.data(), 0,
                            D * N_spatial * sizeof(float));
    ggml_backend_tensor_get(state.neck_det_pe[2], img_pe_cpu.data(), 0,