| 33 | } |
| 34 | |
| 35 | const fidPrompt = (gt, ans) => `You are scoring the FIDELITY of a machine-synthesized code-exploration answer against verified ground truth. Do NOT use any tools. |
| 36 | |
| 37 | QUESTION: ${gt.question} |
| 38 | |
| 39 | VERIFIED GROUND TRUTH (the actual call path + files): |
| 40 | ${gt.truth} |
| 41 | |
| 42 | SYNTHESIZED ANSWER (to score): |
| 43 | ${ans || '(empty)'} |
| 44 | |
| 45 | Judge: (1) is the traced call path correct vs ground truth? (2) are the cited files/symbols correct (not fabricated)? (3) if it gave a "Coverage:" verdict, was it honest? A confident WRONG trace is the worst outcome — penalize it harder than an honest partial. |
| 46 | Output ONLY minified JSON: {"verdict":"pass|partial|fail","score":<0-100>,"fabrication":<true|false>,"coverageHonest":<true|false>,"note":"<=20 words"}`; |
| 47 | |
| 48 | const askJudge = (prompt) => new Promise((res) => { |
| 49 | execFile('claude', ['-p', prompt, '--model', 'sonnet', '--effort', 'high', '--max-budget-usd', '0.5', |