MCPcopy Create free account
hub / github.com/cli/cli / replay

Function replay

evals/src/replay.ts:26–38  ·  view source on GitHub ↗
(path: string)

Source from the content-addressed store, hash-verified

24}
25
26export async function replay(path: string): Promise<ReplayResult> {
27 const absolute = resolve(path);
28 const source = (await stat(absolute)).isDirectory() ? join(absolute, evidenceFile) : absolute;
29 const evidence = parseTrialEvidence(JSON.parse(await readFile(source, "utf8")));
30 const load = async () => evidence;
31 const input = { trajectory: trajectory(source) };
32 // Raw-API stimuli ask for a direct API stream, so, as in their eval.yml, they skip the route and polling checks.
33 const selected = evidence.variant === "raw-api"
34 ? [new EventTaskGrader(load)]
35 : [new EventTaskGrader(load), new EventRouteGrader(load), new NoPollingGrader(load)];
36 const graders = await Promise.all(selected.map(grader => grader.grade(input)));
37 return { path: source, scenario: evidence.scenario, passed: graders.every(result => result.passed), graders };
38}
39
40const usage = `Usage: evals/replay [--json] PATH...
41

Callers 1

replay.test.tsFile · 0.85

Calls 3

parseTrialEvidenceFunction · 0.85
trajectoryFunction · 0.85
gradeMethod · 0.45

Tested by

no test coverage detected