| 52 | * the trial's evidence is recorded for the graders. Trials never overlap. |
| 53 | */ |
| 54 | export class GhAgentExecutor implements Executor { |
| 55 | readonly name = "gh-agent"; |
| 56 | readonly supportsMultiTurn = false; |
| 57 | private context?: Promise<RunContext>; |
| 58 | private queue: Promise<unknown> = Promise.resolve(); |
| 59 | private stopping = false; |
| 60 | |
| 61 | constructor(private readonly suites: SuiteRegistry) {} |
| 62 | |
| 63 | validateConfig(config: unknown): void { |
| 64 | parseExecutorConfig(config, this.suites); |
| 65 | } |
| 66 | |
| 67 | private setup(): Promise<RunContext> { |
| 68 | this.context ??= (async () => { |
| 69 | if (required("GH_EVAL_HOST") !== "github.com") throw new InfrastructureError("This preview suite supports github.com only"); |
| 70 | const org = required("GH_EVAL_ORG"); |
| 71 | if (!/^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/.test(org)) throw new InfrastructureError("Invalid GH_EVAL_ORG"); |
| 72 | const gh = await realpath(required("GH_EVAL_GH")); |
| 73 | await access(gh, constants.R_OK | constants.X_OK); |
| 74 | const token = required("GH_EVAL_TOKEN"); |
| 75 | const configuredBudget = process.env.GH_EVAL_AGENT_BUDGET?.trim(); |
| 76 | const agentBudgetMs = configuredBudget ? effectiveAgentBudget(configuredBudget, undefined, 1) : undefined; |
| 77 | const actorToken = process.env.GH_EVAL_ACTOR_TOKEN?.trim(); |
| 78 | const request: typeof fetch = fetch; |
| 79 | return { |
| 80 | org, gh, token, request, |
| 81 | ...(actorToken ? { actorToken } : {}), |
| 82 | fixtureToken: process.env.GH_EVAL_FIXTURE_TOKEN?.trim() || token, |
| 83 | modelToken: required("COPILOT_GITHUB_TOKEN"), |
| 84 | ...(agentBudgetMs === undefined ? {} : { agentBudgetMs }), |
| 85 | }; |
| 86 | })(); |
| 87 | return this.context; |
| 88 | } |
| 89 | |
| 90 | execute(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> { |
| 91 | // Trials can share one account's limited resources, and a second actor's timing assumes it is the only change in |
| 92 | // flight, so trials never overlap. |
| 93 | const run = this.queue.then(() => { |
| 94 | if (this.stopping) throw new InfrastructureError("The eval run was interrupted before this trial started"); |
| 95 | return this.trial(stimulus, options); |
| 96 | }); |
| 97 | this.queue = run.catch(() => undefined); |
| 98 | return run; |
| 99 | } |
| 100 | |
| 101 | /** Vally calls this on SIGINT or SIGTERM and exits once it resolves, so wait for the running trial to clean up its |
| 102 | * live resources. The trial stops its agent at its next check. */ |
| 103 | async shutdown(): Promise<void> { |
| 104 | this.stopping = true; |
| 105 | await this.queue; |
| 106 | } |
| 107 | |
| 108 | private async trial(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> { |
| 109 | const { suite, scenario } = parseExecutorConfig(options.executorConfig, this.suites); |
| 110 | const context = await this.setup(); |
| 111 | const artifactDir = options.sessionLog?.executorArtifactsDir; |