Function askJudge

scripts/agent-eval/offload-eval-effort.mjs:48–56

(prompt)

Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`;

const askJudge = (prompt) => new Promise((res) => {
  execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5',
    '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
    { cwd: OUT, maxBuffer: 1 << 24, timeout: 120000 }, (err, stdout) => {
      const m = (stdout || '').match(/\{[\s\S]*\}/);
      if (!m) return res({ verdict: 'error', score: null, note: (err ? err.message : 'no json').slice(0, 60) });
      try { res(JSON.parse(m[0])); } catch { res({ verdict: 'error', score: null }); }
    });
});

// ---- 1. Probe: collect synthesized answers at each effort -------------------
const records = [];