| 17 | const CONC = Number(A.concurrency || 4); |
| 18 | |
| 19 | function askJudge(prompt) { |
| 20 | return new Promise((resolve) => { |
| 21 | execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', |
| 22 | '--max-budget-usd', '0.5', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'], |
| 23 | // Run from a neutral dir with no repo files so the judge can't "cheat" by reading source. |
| 24 | { cwd: process.env.AGENT_EVAL_OUT || '/tmp', maxBuffer: 1 << 24, timeout: 120000 }, |
| 25 | (err, stdout) => { |
| 26 | const raw = (stdout || '').trim(); |
| 27 | const m = raw.match(/\{[\s\S]*\}/); |
| 28 | if (!m) return resolve({ verdict: 'error', score: null, note: (err ? 'exec ' + err.message : 'no json').slice(0, 80) }); |
| 29 | try { resolve(JSON.parse(m[0])); } catch { resolve({ verdict: 'error', score: null, note: 'parse fail' }); } |
| 30 | }); |
| 31 | }); |
| 32 | } |
| 33 | |
| 34 | const e2ePrompt = (gt, ans) => `You are scoring whether an AI coding agent correctly answered a code-flow question about a repository. Judge ONLY against the verified ground truth. Do NOT use any tools. |
| 35 | |