Function formatEvalReport

src/eval/harness.ts:166–226

({
  codebaseLabel,
  fixturePath,
  summary,
  redactPaths = true
}: FormatEvalReportParams)

}

export function formatEvalReport({
  codebaseLabel,
  fixturePath,
  summary,
  redactPaths = true
}: FormatEvalReportParams): string {
  const lines: string[] = [];
  const wins = summary.results.filter((result) => result.top1Correct);
  const failures = summary.results.filter((result) => !result.top1Correct);

  lines.push(`\n=== Eval Report: ${codebaseLabel} ===`);
  lines.push(`Fixture: ${fixturePath}`);
  lines.push(
    `Top-1 Accuracy: ${summary.top1Correct}/${summary.total} (${(summary.top1Accuracy * 100).toFixed(0)}%)`
  );
  lines.push(
    `Top-3 Recall:   ${summary.top3RecallCount}/${summary.total} (${(summary.top3Recall * 100).toFixed(0)}%)`
  );
  lines.push(
    `Spec Contamination: ${summary.specContaminatedCount}/${summary.total} (${(summary.specContaminationRate * 100).toFixed(0)}%)`
  );
  lines.push(
    `Gate (${summary.gateThreshold}/${summary.total}): ${summary.passesGate ? 'PASS' : 'FAIL'}`
  );
  lines.push(`Wins: ${wins.length} | Failures: ${failures.length}`);

  lines.push('\nWins:');
  if (wins.length === 0) {
    lines.push('  (none)');
  } else {
    for (const result of wins) {
      lines.push(
        `  PASS #${result.queryId} [${result.category}] "${result.query}" -> ${formatPath(result.topFile, redactPaths)} (${result.score.toFixed(3)})`
      );
    }
  }

  lines.push('\nFailures:');
  if (failures.length === 0) {
    lines.push('  (none)');
  } else {
    for (const result of failures) {
      lines.push(
        `  FAIL #${result.queryId} [${result.category}] "${result.query}" -> ${formatPath(result.topFile, redactPaths)} (${result.score.toFixed(3)})`
      );
      lines.push(`    expected: ${result.expectedPatterns.join(' | ') || '(none)'}`);
      lines.push(`    expected-not: ${result.expectedNotPatterns.join(' | ') || '(none)'}`);
      lines.push('    top-3 actual:');
      if (result.top3Files.length === 0) {
        lines.push('      1. none');
      } else {
        for (let index = 0; index < result.top3Files.length; index++) {
          lines.push(`      ${index + 1}. ${formatPath(result.top3Files[index], redactPaths)}`);
        }
      }
    }
  }