| 324 | } |
| 325 | |
| 326 | async function main() { |
| 327 | if (!USER_TOKEN) { |
| 328 | console.error( |
| 329 | "SLACK_USER_TOKEN missing in .env — run `pnpm exec tsx e2e/grab-user-token.ts` first.", |
| 330 | ); |
| 331 | process.exit(1); |
| 332 | } |
| 333 | |
| 334 | mkdirSync(RESULTS_DIR, { recursive: true }); |
| 335 | const stamp = new Date().toISOString().replace(/[:.]/g, "-"); |
| 336 | const runDir = join(RESULTS_DIR, stamp); |
| 337 | mkdirSync(runDir, { recursive: true }); |
| 338 | |
| 339 | const results: CaseResult[] = []; |
| 340 | // Optional substring filter on case name — convenient when iterating |
| 341 | // on a single case (e.g. `CASE_FILTER='G1 ' pnpm e2e`). |
| 342 | const filter = process.env["CASE_FILTER"]; |
| 343 | const selected = filter |
| 344 | ? CASES.filter((c) => c.name.includes(filter)) |
| 345 | : CASES; |
| 346 | for (const spec of selected) { |
| 347 | process.stdout.write(`\n──── ${spec.name} ────\n`); |
| 348 | try { |
| 349 | const r = await runCase(spec); |
| 350 | const flag = r.status === "pass" ? "✓" : "✗"; |
| 351 | console.log( |
| 352 | ` ${flag} ${r.durationMs}ms len=${r.finalText?.length ?? 0} samples=${r.samples.length} unbalanced=${r.unbalancedSamples}`, |
| 353 | ); |
| 354 | if (r.errors.length) console.log(" " + r.errors.join("\n ")); |
| 355 | if (r.followUp) { |
| 356 | const fflag = r.followUp.status === "pass" ? "✓" : "✗"; |
| 357 | console.log( |
| 358 | ` ↳ followUp ${fflag} ${r.followUp.durationMs}ms len=${r.followUp.finalText?.length ?? 0} samples=${r.followUp.samples.length} unbalanced=${r.followUp.unbalancedSamples}`, |
| 359 | ); |
| 360 | if (r.followUp.errors.length) |
| 361 | console.log(" " + r.followUp.errors.join("\n ")); |
| 362 | } |
| 363 | if (r.interrupt) { |
| 364 | console.log( |
| 365 | ` ↳ interrupt first_len=${r.interrupt.firstReplyText?.length ?? 0} second_len=${r.interrupt.secondReplyText?.length ?? 0}`, |
| 366 | ); |
| 367 | if (r.interrupt.errors.length) |
| 368 | console.log(" " + r.interrupt.errors.join("\n ")); |
| 369 | } |
| 370 | results.push(r); |
| 371 | } catch (err) { |
| 372 | console.log(` ✗ exception: ${(err as Error).message}`); |
| 373 | results.push({ |
| 374 | name: spec.name, |
| 375 | prompt: spec.prompt, |
| 376 | status: "fail", |
| 377 | errors: [(err as Error).message], |
| 378 | durationMs: 0, |
| 379 | finalText: undefined, |
| 380 | unbalancedSamples: 0, |
| 381 | samples: [], |
| 382 | }); |
| 383 | } |