MCPcopy Create free account
hub / github.com/cli/cli / analyse

Function analyse

evals/src/suites/events/grading/analysis.ts:24–92  ·  view source on GitHub ↗
(evidence: Evidence)

Source from the content-addressed store, hash-verified

22
23/** Mechanical facts shared by the route and polling graders, computed once from recorded evidence. */
24export function analyse(evidence: Evidence): RouteAnalysis {
25 const { scenario, calls, snapshot } = evidence;
26 const changes = snapshot.actions.filter(action => action.kind !== "fix");
27 const observations = changes.flatMap((action, index): Observation[] => {
28 const step = scenarios[scenario].steps[index];
29 if (!step || step.kind !== action.kind) return [];
30 const delivery = "delivery" in step ? step.delivery : undefined;
31 const recovery = delivery === "after-tool" || delivery === "after-fix";
32 const trigger = delivery === "after-tool" ? "after-tool" : delivery === "after-fix" ? "after-fix" : "wait";
33 // Recovery steps are graded on outcome, so a late fallback delivery is held to the same evidence.
34 if (action.trigger !== trigger && !(recovery && action.trigger === "fallback")) return [];
35 // The actor acts during one listener, but an agent may run several bound to different events, so any listener
36 // running at the change may deliver it. The actor's own listener is credited first.
37 const candidates = [...calls].sort((a, b) => Number(b.id === action.callId) - Number(a.id === action.callId));
38 for (const call of candidates) {
39 if (call.kind !== "gh" || (!recovery && call.startedAt > action.at)
40 || (call.endedAt ?? snapshot.observedUntil) < action.at) continue;
41 const at = observedAt(evidence, action, call);
42 if (at !== undefined) {
43 return [{ callId: call.id, startedAt: call.startedAt, at, action: action.kind, recovered: call.startedAt > action.at }];
44 }
45 }
46 if (recovery) {
47 // A review posted outside any listener is graded on outcome: any status read that shows it proves nothing
48 // was missed, whether the read ran before, alongside or after a listener.
49 const expected = snapshot.reviews.find(review => review.id === action.reviewId);
50 for (const read of calls) {
51 if (read.kind !== "gh" || !statusRead(read)) continue;
52 const at = reviewReadAt(read, action, expected);
53 if (at !== undefined && at >= action.at) {
54 return [{ callId: read.id, startedAt: read.startedAt, at, action: action.kind, recovered: true }];
55 }
56 }
57 }
58 return [];
59 });
60 const waitCalls = calls.filter(call => dedicatedWait(call));
61 const waits = waitCalls.map(call => call.startedAt);
62 // A status read is polling when it repeats the same projection with no intervening external change or
63 // listener boundary. Actor changes, listener starts, stream attachments and listener exits each open a new
64 // epoch, so one read per change is allowed, as are the reads the hint and the listener ask for: one once a
65 // stream is attached, including after each reconnect, to catch changes made while none was, and one after the
66 // listener exits, to see what a thin event changed.
67 const boundaries = [
68 ...changes.map(change => change.at),
69 ...waits,
70 ...waitCalls.flatMap(call => attachments(call).at),
71 ...waitCalls.flatMap(call => call.endedAt === undefined ? [] : [call.endedAt]),
72 ];
73 const reads = new Set<string>();
74 let statusReads = 0;
75 const polling = [...calls].sort((a, b) => a.startedAt - b.startedAt).filter(call => {
76 const projection = statusRead(call);
77 if (!projection) return false;
78 statusReads++;
79 const epoch = boundaries.filter(at => at <= call.startedAt).length;
80 const key = `${epoch}:${projection}`;
81 if (reads.has(key)) return true;

Callers 2

gradeMethod · 0.85
gradeMethod · 0.85

Calls 6

observedAtFunction · 0.85
statusReadFunction · 0.85
reviewReadAtFunction · 0.85
dedicatedWaitFunction · 0.85
attachmentsFunction · 0.85
findMethod · 0.80

Tested by

no test coverage detected