Function runTask

eval/run.mjs:108–151

(task)

}

async function runTask(task) {
  const work = await fs.mkdtemp(path.join(os.tmpdir(), `qodex-eval-${task.id}-`));
  try {
    for (const [rel, content] of Object.entries(task.setup?.files ?? {})) {
      const abs = path.join(work, rel);
      await fs.mkdir(path.dirname(abs), { recursive: true });
      await fs.writeFile(abs, content, 'utf-8');
    }

    // A TypeScript task needs tsc reachable via `npx --no-install tsc` (what both the
    // auto-verify gate and the check command run). Real projects have it in node_modules;
    // our throwaway workspace doesn't, so symlink this repo's node_modules in. Contained
    // to the harness — product code is untouched.
    if (fsSync.existsSync(path.join(work, 'tsconfig.json'))) {
      try {
        await fs.symlink(path.join(repoRoot, 'node_modules'), path.join(work, 'node_modules'), 'dir');
      } catch { /* best-effort */ }
    }

    const started = Date.now();
    const run = await runAgent(task.prompt, work);
    const wallMs = Date.now() - started;
    const tele = parseAgentEvents(run.stdout);

    const outcome = await observeOutcome(task, work, tele.error);
    const verdict = evaluateChecks(task.check, outcome);

    process.stdout.write(`  ${verdict.passed ? '✅' : '❌'} ${task.id}  (${tele.iterations} iters, ${tele.toolCalls} tools, ${(wallMs / 1000).toFixed(1)}s)\n`);
    if (!verdict.passed) for (const r of verdict.reasons) process.stdout.write(`       · ${r}\n`);

    return {
      id: task.id,
      passed: verdict.passed,
      reasons: verdict.reasons,
      iterations: tele.iterations,
      toolCalls: tele.toolCalls,
      costUsd: tele.costUsd,
      wallMs,
      error: tele.error,
    };
  } finally {
    await fs.rm(work, { recursive: true, force: true }).catch(() => {});
  }
}

async function main() {
  const filter = process.argv.slice(2);