| 151 | } |
| 152 | |
| 153 | async function main() { |
| 154 | const filter = process.argv.slice(2); |
| 155 | const tasks = await loadTasks(filter); |
| 156 | if (tasks.length === 0) { |
| 157 | console.error(`No tasks found in ${tasksDir}${filter.length ? ` matching ${filter.join(', ')}` : ''}.`); |
| 158 | process.exit(1); |
| 159 | } |
| 160 | |
| 161 | console.log(`Running ${tasks.length} eval task(s)${model ? ` with model ${model}` : ''}…\n`); |
| 162 | const results = []; |
| 163 | for (const task of tasks) { |
| 164 | results.push(await runTask(task)); |
| 165 | } |
| 166 | |
| 167 | const summary = summarize(results); |
| 168 | const when = new Date().toISOString(); |
| 169 | const report = formatReport(results, summary, { model: model ?? '(default)', when }); |
| 170 | |
| 171 | const reportPath = path.join(repoRoot, 'eval', 'report.md'); |
| 172 | await fs.writeFile(reportPath, report, 'utf-8'); |
| 173 | |
| 174 | console.log(`\n${summary.passed}/${summary.total} passed (${(summary.passRate * 100).toFixed(0)}%). Report: ${path.relative(repoRoot, reportPath)}`); |
| 175 | } |
| 176 | |
| 177 | main().catch(err => { console.error('Eval harness error:', err); process.exit(1); }); |