* Evaluate a diff pair directly using the judge's LLM. * This bypasses git operations and uses static diffs for consistency testing.
( judge: Judge, systemPrompt: string, userPrompt: string, )
| 38 | * This bypasses git operations and uses static diffs for consistency testing. |
| 39 | */ |
| 40 | async function evaluateWithJudge( |
| 41 | judge: Judge, |
| 42 | systemPrompt: string, |
| 43 | userPrompt: string, |
| 44 | ): Promise<ScoreResult> { |
| 45 | try { |
| 46 | const { object } = await generateObject({ |
| 47 | model: judge.model, |
| 48 | schema: scoreResultSchema, |
| 49 | system: systemPrompt, |
| 50 | temperature: 0, |
| 51 | prompt: userPrompt, |
| 52 | }); |
| 53 | |
| 54 | return object; |
| 55 | } catch (error) { |
| 56 | const message = error instanceof Error ? error.message : String(error); |
| 57 | throw new Error(`Judge evaluation failed: ${message}`); |
| 58 | } |
| 59 | } |
| 60 | |
| 61 | /** |
| 62 | * Helper to create the user prompt for diff comparison. |