(path: string)
| 24 | } |
| 25 | |
| 26 | export async function replay(path: string): Promise<ReplayResult> { |
| 27 | const absolute = resolve(path); |
| 28 | const source = (await stat(absolute)).isDirectory() ? join(absolute, evidenceFile) : absolute; |
| 29 | const evidence = parseTrialEvidence(JSON.parse(await readFile(source, "utf8"))); |
| 30 | const load = async () => evidence; |
| 31 | const input = { trajectory: trajectory(source) }; |
| 32 | // Raw-API stimuli ask for a direct API stream, so, as in their eval.yml, they skip the route and polling checks. |
| 33 | const selected = evidence.variant === "raw-api" |
| 34 | ? [new EventTaskGrader(load)] |
| 35 | : [new EventTaskGrader(load), new EventRouteGrader(load), new NoPollingGrader(load)]; |
| 36 | const graders = await Promise.all(selected.map(grader => grader.grade(input))); |
| 37 | return { path: source, scenario: evidence.scenario, passed: graders.every(result => result.passed), graders }; |
| 38 | } |
| 39 | |
| 40 | const usage = `Usage: evals/replay [--json] PATH... |
| 41 |
no test coverage detected