Function evaluateDiscoveryTask

src/eval/discovery-harness.ts:172–207

(
  task: DiscoveryTaskType,
  result: DiscoverySurfaceResultType,
  elapsedMs?: number
)

}

function evaluateDiscoveryTask(
  task: DiscoveryTaskType,
  result: DiscoverySurfaceResultType,
  elapsedMs?: number
): DiscoveryTaskResultEval {
  const { matchedSignals, missingSignals, forbiddenHits, usefulnessScore } = matchSignals(
    result.payload,
    task.expectedSignals,
    task.forbiddenSignals
  );
  const payloadBytes = countUtf8Bytes(result.payload);
  const estimatedTokens = estimateTokenCountFromBytes(payloadBytes);
  const firstRelevantHit = matchPatterns(result.topFiles ?? [], task.expectedFilePatterns);
  const bestExampleUseful =
    task.expectedBestExamplePatterns && task.expectedBestExamplePatterns.length > 0
      ? task.expectedBestExamplePatterns.some((pattern) =>
          normalizeText(result.bestExample ?? '').includes(normalizeText(pattern))
        )
      : undefined;

  return {
    taskId: task.id,
    title: task.title,
    job: task.job,
    surface: task.surface,
    usefulnessScore,
    matchedSignals,
    missingSignals,
    forbiddenHits,
    payloadBytes,
    estimatedTokens,
    ...(firstRelevantHit !== null ? { firstRelevantHit } : {}),
    ...(typeof bestExampleUseful === 'boolean' ? { bestExampleUseful } : {}),
    ...(typeof elapsedMs === 'number' ? { elapsedMs, toolCallCount: 1 } : {})
  };
}

function summarizeDiscoveryResults(results: DiscoveryTaskResultEval[]): DiscoverySummaryType {
  const totalTasks = results.length;