MCPcopy Create free account
hub / github.com/cli/cli / GhAgentExecutor

Class GhAgentExecutor

evals/src/core/executor.ts:54–267  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

52 * the trial's evidence is recorded for the graders. Trials never overlap.
53 */
54export class GhAgentExecutor implements Executor {
55 readonly name = "gh-agent";
56 readonly supportsMultiTurn = false;
57 private context?: Promise<RunContext>;
58 private queue: Promise<unknown> = Promise.resolve();
59 private stopping = false;
60
61 constructor(private readonly suites: SuiteRegistry) {}
62
63 validateConfig(config: unknown): void {
64 parseExecutorConfig(config, this.suites);
65 }
66
67 private setup(): Promise<RunContext> {
68 this.context ??= (async () => {
69 if (required("GH_EVAL_HOST") !== "github.com") throw new InfrastructureError("This preview suite supports github.com only");
70 const org = required("GH_EVAL_ORG");
71 if (!/^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/.test(org)) throw new InfrastructureError("Invalid GH_EVAL_ORG");
72 const gh = await realpath(required("GH_EVAL_GH"));
73 await access(gh, constants.R_OK | constants.X_OK);
74 const token = required("GH_EVAL_TOKEN");
75 const configuredBudget = process.env.GH_EVAL_AGENT_BUDGET?.trim();
76 const agentBudgetMs = configuredBudget ? effectiveAgentBudget(configuredBudget, undefined, 1) : undefined;
77 const actorToken = process.env.GH_EVAL_ACTOR_TOKEN?.trim();
78 const request: typeof fetch = fetch;
79 return {
80 org, gh, token, request,
81 ...(actorToken ? { actorToken } : {}),
82 fixtureToken: process.env.GH_EVAL_FIXTURE_TOKEN?.trim() || token,
83 modelToken: required("COPILOT_GITHUB_TOKEN"),
84 ...(agentBudgetMs === undefined ? {} : { agentBudgetMs }),
85 };
86 })();
87 return this.context;
88 }
89
90 execute(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> {
91 // Trials can share one account's limited resources, and a second actor's timing assumes it is the only change in
92 // flight, so trials never overlap.
93 const run = this.queue.then(() => {
94 if (this.stopping) throw new InfrastructureError("The eval run was interrupted before this trial started");
95 return this.trial(stimulus, options);
96 });
97 this.queue = run.catch(() => undefined);
98 return run;
99 }
100
101 /** Vally calls this on SIGINT or SIGTERM and exits once it resolves, so wait for the running trial to clean up its
102 * live resources. The trial stops its agent at its next check. */
103 async shutdown(): Promise<void> {
104 this.stopping = true;
105 await this.queue;
106 }
107
108 private async trial(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> {
109 const { suite, scenario } = parseExecutorConfig(options.executorConfig, this.suites);
110 const context = await this.setup();
111 const artifactDir = options.sessionLog?.executorArtifactsDir;

Callers

nothing calls this directly

Calls 1

resolveMethod · 0.65

Tested by

no test coverage detected