Function formatEditPreflightReport

src/eval/edit-preflight-harness.ts:195–269

({
  codebaseLabel,
  fixturePath,
  summary
}: FormatEditPreflightReportParams)

}

export function formatEditPreflightReport({
  codebaseLabel,
  fixturePath,
  summary
}: FormatEditPreflightReportParams): string {
  const lines: string[] = [];
  const unsafeFalsePositives = summary.results.filter(
    (result) => result.risk === 'unsafe' && result.ready
  );
  const safeMisses = summary.results.filter((result) => result.risk === 'safe' && !result.ready);

  lines.push(`\n=== Edit Preflight Eval Report: ${codebaseLabel} ===`);
  lines.push(`Fixture: ${fixturePath}`);
  lines.push(
    `Tasks: ${summary.totalTasks} (${summary.safeTasks} safe, ${summary.unsafeTasks} unsafe)`
  );
  lines.push(
    `Top-target in top-3: ${summary.topTargetInTop3Count}/${summary.targetableTasks} (${formatRate(summary.topTargetInTop3Rate)})`
  );
  lines.push(`Average first relevant hit: ${formatHit(summary.averageFirstRelevantHit)}`);
  lines.push(
    `Best-example hit rate: ${summary.bestExampleHitCount}/${summary.bestExampleTasks} (${formatRate(summary.bestExampleHitRate)})`
  );
  lines.push(
    `Safe-task ready rate: ${summary.safeTaskReadyCount}/${summary.safeTasks} (${formatRate(summary.safeTaskReadyRate)})`
  );
  lines.push(
    `Unsafe-task abstain rate: ${summary.unsafeTaskAbstainCount}/${summary.unsafeTasks} (${formatRate(summary.unsafeTaskAbstainRate)})`
  );
  lines.push(
    `Unsafe ready=true false-positive rate: ${summary.unsafeReadyFalsePositiveCount}/${summary.unsafeTasks} (${formatRate(summary.unsafeReadyFalsePositiveRate)})`
  );
  lines.push('');
  lines.push('Task results:');

  for (const result of summary.results) {
    const taskLine = [
      `- ${result.taskId}`,
      `[${result.risk}]`,
      `ready=${result.ready ? 'yes' : 'no'}`,
      `abstain=${result.abstain ? 'yes' : 'no'}`,
      `firstRelevant=${result.firstRelevantHit ?? 'n/a'}`,
      `top3=${result.topTargetInTop3 === null ? 'n/a' : result.topTargetInTop3 ? 'hit' : 'miss'}`,
      `bestExample=${result.bestExampleHit === null ? 'n/a' : result.bestExampleHit ? 'hit' : 'miss'}`,
      `quality=${result.searchQualityStatus}`
    ];
    lines.push(taskLine.join(' '));
  }

  lines.push('');
  lines.push('Unsafe false positives:');
  if (unsafeFalsePositives.length === 0) {
    lines.push('  (none)');
  } else {
    for (const result of unsafeFalsePositives) {
      lines.push(`  - ${result.taskId}: "${result.query}"`);
    }
  }