Function evaluateWithJudge

tests/judgeConsistencyMedium.test.ts:38–57

* Medium Complexity Judge Consistency Tests * * These tests verify that judges produce consistent scores on more complex scenarios: * - Nested loops with error handling * - State mutations across methods * - Multi-file API changes * - Import reorganizations * * All tests run 3 evaluations an

(
  judge: Judge,
  systemPrompt: string,
  userPrompt: string,
)

 */

async function evaluateWithJudge(
  judge: Judge,
  systemPrompt: string,
  userPrompt: string,
): Promise<ScoreResult> {
  try {
    const { object } = await generateObject({
      model: judge.model,
      schema: scoreResultSchema,
      system: systemPrompt,
      temperature: 0,
      prompt: userPrompt,
    });

    return object;
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    throw new Error(`Judge evaluation failed: ${message}`);
  }
}

function createDiffComparisonPrompt(
  diffPair: DiffPair,