()
| 281 | // ── Main ─────────────────────────────────────────────────────────────────────── |
| 282 | |
| 283 | async function main() { |
| 284 | mkdirSync(RESULTS_DIR, { recursive: true }); |
| 285 | const stamp = new Date().toISOString().replace(/[:.]/g, "-"); |
| 286 | const runDir = join(RESULTS_DIR, stamp); |
| 287 | mkdirSync(runDir, { recursive: true }); |
| 288 | |
| 289 | const results: CaseResult[] = []; |
| 290 | const filter = process.env["CASE_FILTER"]; |
| 291 | const selected = filter |
| 292 | ? CASES.filter((c) => c.name.includes(filter)) |
| 293 | : CASES; |
| 294 | |
| 295 | for (const spec of selected) { |
| 296 | process.stdout.write(`\n──── ${spec.name} ────\n`); |
| 297 | try { |
| 298 | const r = await runCase(spec); |
| 299 | const flag = r.status === "pass" ? "✓" : "✗"; |
| 300 | console.log( |
| 301 | ` ${flag} ${r.durationMs}ms len=${r.finalText?.length ?? 0} samples=${r.samples.length}`, |
| 302 | ); |
| 303 | if (r.errors.length) console.log(" " + r.errors.join("\n ")); |
| 304 | if (r.followUp) { |
| 305 | const fflag = r.followUp.status === "pass" ? "✓" : "✗"; |
| 306 | console.log( |
| 307 | ` ↳ followUp ${fflag} ${r.followUp.durationMs}ms len=${r.followUp.finalText?.length ?? 0} samples=${r.followUp.samples.length}`, |
| 308 | ); |
| 309 | if (r.followUp.errors.length) { |
| 310 | console.log(" " + r.followUp.errors.join("\n ")); |
| 311 | } |
| 312 | } |
| 313 | results.push(r); |
| 314 | } catch (err) { |
| 315 | console.log(` ✗ exception: ${(err as Error).message}`); |
| 316 | results.push({ |
| 317 | name: spec.name, |
| 318 | prompt: spec.prompt, |
| 319 | status: "fail", |
| 320 | errors: [(err as Error).message], |
| 321 | durationMs: 0, |
| 322 | finalText: undefined, |
| 323 | samples: [], |
| 324 | }); |
| 325 | } |
| 326 | } |
| 327 | |
| 328 | writeFileSync( |
| 329 | join(runDir, "report.json"), |
| 330 | JSON.stringify( |
| 331 | { ranAt: stamp, mode: AUTOMATED ? "automated" : "manual", results }, |
| 332 | null, |
| 333 | 2, |
| 334 | ), |
| 335 | ); |
| 336 | const pass = results.filter((r) => r.status === "pass").length; |
| 337 | console.log( |
| 338 | `\n${pass}/${results.length} cases passed. Report: ${runDir}/report.json`, |
| 339 | ); |
| 340 | process.exit(pass === results.length ? 0 : 1); |
no test coverage detected