Function evaluateAndShow

scripts/show-test-outputs.ts:16–47

(
  name: string,
  systemPrompt: string,
  reference: string,
  candidate: string,
  scoreType: string,
)

const judge = judges[0]; // claude-4.5

async function evaluateAndShow(
  name: string,
  systemPrompt: string,
  reference: string,
  candidate: string,
  scoreType: string,
) {
  console.log(`\n${"=".repeat(80)}`);
  console.log(`TEST: ${name}`);
  console.log(`${"=".repeat(80)}\n`);

  const userPrompt =
    scoreType === "logic"
      ? `Reference diff:\n${reference}\n\nCandidate diff:\n${candidate}\n\nCompare ONLY the logical behavior (conditions, edge cases, side effects). Ignore code structure and style. Respond with JSON.`
      : `Reference diff:\n${reference}\n\nCandidate diff:\n${candidate}\n\nCompare ONLY the API signatures (function names, parameter order, parameter names). Ignore implementation details. Respond with JSON.`;

  for (let i = 1; i <= 3; i++) {
    console.log(`\n--- Run ${i} ---\n`);

    const { object } = await generateObject({
      model: judge.model,
      schema: scoreResultSchema,
      system: systemPrompt,
      temperature: 0,
      prompt: userPrompt,
    });

    console.log(`Score: ${object.score}`);
    console.log(`\nRationale:\n${object.rationale}`);
    console.log();
  }
}

// Run tests
(async () => {