Function testConsistency

tests/judgeConsistencyComplex.test.ts:57–73

(
  judge: Judge,
  systemPrompt: string,
  diffPair: DiffPair,
  scoreType: "logic-equivalence",
  runs: number = 3,
)

}

async function testConsistency(
  judge: Judge,
  systemPrompt: string,
  diffPair: DiffPair,
  scoreType: "logic-equivalence",
  runs: number = 3,
): Promise<ScoreResult[]> {
  const userPrompt = createDiffComparisonPrompt(diffPair, scoreType);

  const results = await Promise.all(
    Array.from({ length: runs }, () =>
      evaluateWithJudge(judge, systemPrompt, userPrompt),
    ),
  );

  return results;
}

const testJudge = judges[0];