Function run

__tests__/evaluation/runner.ts:33–118

()

console.log('');

async function run() {
  const cg = CodeGraph.openSync(resolvedPath);
  const results: EvalResult[] = [];

  for (const tc of testCases) {
    const start = performance.now();

    if (tc.api === 'searchNodes') {
      const searchResults = cg.searchNodes(tc.query, {
        limit: 10,
        kinds: tc.kinds,
        ...(tc.options as Record<string, unknown>),
      });
      const latency = performance.now() - start;
      const result = scoreSearchNodes(tc.id, tc.expectedSymbols, searchResults, latency);
      results.push(result);
    } else {
      const subgraph = await cg.findRelevantContext(tc.query, {
        searchLimit: 8,
        traversalDepth: 3,
        maxNodes: 80,
        minScore: 0.2,
        ...(tc.options as Record<string, unknown>),
      });
      const latency = performance.now() - start;
      const result = scoreFindRelevantContext(tc.id, tc.expectedSymbols, subgraph, latency);
      results.push(result);
    }
  }

  cg.close();

  // Print results table
  const maxIdLen = Math.max(...results.map((r) => r.caseId.length));

  for (const r of results) {
    const status = r.pass ? '\x1b[32mPASS\x1b[0m' : '\x1b[31mFAIL\x1b[0m';
    const id = r.caseId.padEnd(maxIdLen);
    const recall = `recall=${r.recall.toFixed(2)}`;
    const extra =
      r.edgeDensity !== undefined
        ? `density=${r.edgeDensity.toFixed(2)}`
        : `mrr=${r.mrr.toFixed(2)}`;
    const latency = `${Math.round(r.latencyMs)}ms`;

    console.log(`  ${id}  ${status}  ${recall}  ${extra}  ${latency}`);

    if (r.missedSymbols.length > 0) {
      console.log(`  ${' '.repeat(maxIdLen)}        missed: ${r.missedSymbols.join(', ')}`);
    }
  }

  // Summary
  const passed = results.filter((r) => r.pass).length;
  const failed = results.length - passed;
  const meanRecall = results.reduce((s, r) => s + r.recall, 0) / results.length;
  const mrrResults = results.filter((r) => r.mrr > 0 || r.caseId.startsWith('search-'));
  const meanMRR =