MCPcopy Create free account
hub / github.com/cli/cli / start

Function start

evals/src/suites/events/suite.ts:24–67  ·  view source on GitHub ↗
(context: TrialContext)

Source from the content-addressed store, hash-verified

22}
23
24async function start(context: TrialContext): Promise<Trial> {
25 const { run, workDir, env, journalPath, progress } = context;
26 const scenario = context.scenario as Scenario;
27 if (!run.actorToken) throw new InfrastructureError("GH_EVAL_ACTOR_TOKEN must be supplied explicitly");
28 const github = new EventsGitHub(run.fixtureToken, run.request);
29 const subscriber = new EventsGitHub(run.token, run.request);
30 const definition: ScenarioDefinition = scenarios[scenario];
31 const seeding = definition.fixture.seedSubscriptions === true;
32 // Seeded subscriptions must be the only ones, so the grader knows what each deletion removed.
33 if (seeding && (await subscriber.subscriptions()).length) {
34 throw new InfrastructureError("The GH_EVAL_TOKEN user must start this scenario with no event subscriptions. "
35 + "Clean up leftover subscriptions before running again.");
36 }
37 progress("creating sandbox repository and fixture");
38 const { fixture } = await provision(github, run.org, scenario, workDir, env, journalPath, subscriber);
39 if (fixture.seeded) {
40 progress(`seeded ${fixture.seeded.stale.length} stale and ${fixture.seeded.active.length} active subscriptions`);
41 }
42 const actor = new Actor(
43 github, new EventsGitHub(run.actorToken, run.request), fixture, context.agentBudgetMs / 3, Date.now());
44 const trialActor: TrialActor<Action> = {
45 get done() { return actor.done; },
46 actions: actor.actions,
47 postTool: definition.steps.some(step => "delivery" in step
48 && (step.delivery === "after-tool" || step.delivery === "after-fix")),
49 trigger: (calls, now) => calls.find(call => call.endedAt === undefined && dedicatedWait(call) !== undefined
50 && readyForActor(call, calls as Call[], now)),
51 triggerNote: "recognised an attached dedicated gh wait from its recorded argv and stderr",
52 tick: (now, trigger, source) => actor.tick(now, trigger, source),
53 describe,
54 };
55 return {
56 placeholders: fixture.pr ? { PR_URL: fixture.pr.html_url } : {},
57 summary: fixture.pr?.html_url ?? `https://github.com/${fixture.repo} (agent creates the PR)`,
58 actor: trialActor as TrialActor,
59 validate(rawEvents) {
60 if (!seeding && subscriptionLimitHit(rawEvents as unknown[])) {
61 throw new InfrastructureError("The GH_EVAL_TOKEN user hit the event subscription limit, so this "
62 + "trial cannot be graded. Clean up leftover subscriptions before running again.");
63 }
64 },
65 snapshot: observedUntil => captureSnapshot(github, subscriber, fixture, workDir, env, actor.actions, observedUntil),
66 };
67}
68
69async function captureSnapshot(
70 github: EventsGitHub, subscriber: EventsGitHub, fixture: Pick<Fixture, "repo" | "baseSha" | "pr" | "seeded">,

Callers

nothing calls this directly

Calls 7

subscriptionsMethod · 0.95
tickMethod · 0.95
provisionFunction · 0.85
dedicatedWaitFunction · 0.85
readyForActorFunction · 0.85
captureSnapshotFunction · 0.85
findMethod · 0.80

Tested by

no test coverage detected