MCPcopy Create free account
hub / github.com/colbymchenry/codegraph / askJudge

Function askJudge

scripts/agent-eval/offload-eval-judge.mjs:19–32  ·  view source on GitHub ↗
(prompt)

Source from the content-addressed store, hash-verified

17const CONC = Number(A.concurrency || 4);
18
19function askJudge(prompt) {
20 return new Promise((resolve) => {
21 execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high',
22 '--max-budget-usd', '0.5', '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}'],
23 // Run from a neutral dir with no repo files so the judge can't "cheat" by reading source.
24 { cwd: process.env.AGENT_EVAL_OUT || '/tmp', maxBuffer: 1 << 24, timeout: 120000 },
25 (err, stdout) => {
26 const raw = (stdout || '').trim();
27 const m = raw.match(/\{[\s\S]*\}/);
28 if (!m) return resolve({ verdict: 'error', score: null, note: (err ? 'exec ' + err.message : 'no json').slice(0, 80) });
29 try { resolve(JSON.parse(m[0])); } catch { resolve({ verdict: 'error', score: null, note: 'parse fail' }); }
30 });
31 });
32}
33
34const e2ePrompt = (gt, ans) => `You are scoring whether an AI coding agent correctly answered a code-flow question about a repository. Judge ONLY against the verified ground truth. Do NOT use any tools.
35

Callers 1

workerFunction · 0.70

Calls 2

parseMethod · 0.80
resolveFunction · 0.50

Tested by

no test coverage detected