Function callLocal

src/tools/vision/vision-analyze.ts:277–332

(base64: string, mime: string, prompt: string)

}

async function callLocal(base64: string, mime: string, prompt: string): Promise<string> {
  // CRITICAL: only call this if the user has EXPLICITLY configured a vision model.
  // Otherwise the request hits whatever text model is currently loaded in LM Studio
  // (e.g. Qwen3-Coder, which is text-only), and the model returns a confused
  // "I can't see images" response that LOOKS like a real analysis to the parent agent.
  //
  // We therefore require QODEX_LOCAL_VISION_MODEL to be set. The default LM Studio
  // port is fine, but the model id MUST be explicit — there's no way for us to
  // tell from the response alone whether the loaded model supports vision.
  const explicitModel = resolveLocalVisionModel();
  if (!explicitModel) {
    throw new Error(
      'Local vision backend not configured. Set QODEX_LOCAL_VISION_MODEL to the id ' +
      'of a vision-capable model loaded in LM Studio (e.g. "qwen2.5-vl-7b" or "qwen3-vl-32b"), ' +
      'point roles.vision at an openai-provider model, or run a vision-capable primary model. ' +
      'Without this, requests would hit your text model and return fake analysis.',
    );
  }
  const baseUrl = process.env.QODEX_LOCAL_VISION_URL ?? 'http://127.0.0.1:1234/v1';
  const res = await fetch(`${baseUrl}/chat/completions`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: explicitModel,
      max_tokens: 1024,
      messages: [
        {
          role: 'user',
          content: [
            { type: 'text', text: prompt },
            { type: 'image_url', image_url: { url: `data:${mime};base64,${base64}` } },
          ],
        },
      ],
    }),
  });
  if (!res.ok) {
    const errText = await res.text().catch(() => '');
    throw new Error(`Local vision backend HTTP ${res.status}: ${errText.slice(0, 300)}. Ensure a vision-capable model is loaded in LM Studio.`);
  }
  const body = await res.json() as { choices?: Array<{ message?: { content?: string } }> };
  const content = body.choices?.[0]?.message?.content ?? '';

  // Heuristic: if the response sounds like a text-only model apologizing about not
  // being able to see images, treat it as a backend failure so the chain falls through.
  // This catches the case where the user set QODEX_LOCAL_VISION_MODEL but actually
  // a non-vision model is loaded under that id (LM Studio doesn't enforce capabilities).
  if (looksLikeTextOnlyRefusal(content)) {
    throw new Error(
      `Local model '${explicitModel}' responded as if it can't see images. ` +
      `Verify a real vision model is loaded in LM Studio (Qwen2.5-VL, Qwen3-VL, etc) — ` +
      `the model id alone isn't enough; the loaded weights must support vision.`,
    );
  }
  return content;
}

/**