Function evaluateQuery

src/eval/harness.ts:56–90

(query: EvalQuery, results: SearchResult[])

}

function evaluateQuery(query: EvalQuery, results: SearchResult[]): EvalResult {
  const uniqueResults = dedupeByFile(results);
  const topFile = uniqueResults.length > 0 ? uniqueResults[0].filePath : null;
  const top3Files = uniqueResults.slice(0, 3).map((result) => result.filePath);

  const expectedPatterns = getExpectedPatterns(query);
  const expectedNotPatterns = getExpectedNotPatterns(query);

  const top1Correct =
    topFile !== null &&
    matchesPattern(topFile, expectedPatterns) &&
    !matchesPattern(topFile, expectedNotPatterns);

  const top3Recall = top3Files.some(
    (filePath) =>
      matchesPattern(filePath, expectedPatterns) && !matchesPattern(filePath, expectedNotPatterns)
  );

  const specCount = top3Files.filter((filePath) => isTestFile(filePath)).length;
  const specContaminated = specCount >= 2;

  return {
    queryId: query.id,
    query: query.query,
    category: query.category,
    expectedPatterns,
    expectedNotPatterns,
    topFile,
    top3Files,
    top1Correct,
    top3Recall,
    specContaminated,
    score: uniqueResults.length > 0 ? uniqueResults[0].score : 0
  };
}

function resolveGateThreshold(total: number, gate: EvalGate): number {
  if (gate <= 1) {