| 65 | } |
| 66 | |
| 67 | private setup(): Promise<RunContext> { |
| 68 | this.context ??= (async () => { |
| 69 | if (required("GH_EVAL_HOST") !== "github.com") throw new InfrastructureError("This preview suite supports github.com only"); |
| 70 | const org = required("GH_EVAL_ORG"); |
| 71 | if (!/^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/.test(org)) throw new InfrastructureError("Invalid GH_EVAL_ORG"); |
| 72 | const gh = await realpath(required("GH_EVAL_GH")); |
| 73 | await access(gh, constants.R_OK | constants.X_OK); |
| 74 | const token = required("GH_EVAL_TOKEN"); |
| 75 | const configuredBudget = process.env.GH_EVAL_AGENT_BUDGET?.trim(); |
| 76 | const agentBudgetMs = configuredBudget ? effectiveAgentBudget(configuredBudget, undefined, 1) : undefined; |
| 77 | const actorToken = process.env.GH_EVAL_ACTOR_TOKEN?.trim(); |
| 78 | const request: typeof fetch = fetch; |
| 79 | return { |
| 80 | org, gh, token, request, |
| 81 | ...(actorToken ? { actorToken } : {}), |
| 82 | fixtureToken: process.env.GH_EVAL_FIXTURE_TOKEN?.trim() || token, |
| 83 | modelToken: required("COPILOT_GITHUB_TOKEN"), |
| 84 | ...(agentBudgetMs === undefined ? {} : { agentBudgetMs }), |
| 85 | }; |
| 86 | })(); |
| 87 | return this.context; |
| 88 | } |
| 89 | |
| 90 | execute(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> { |
| 91 | // Trials can share one account's limited resources, and a second actor's timing assumes it is the only change in |