(stimulus: Stimulus, options: ExecutorOptions)
| 106 | } |
| 107 | |
| 108 | private async trial(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> { |
| 109 | const { suite, scenario } = parseExecutorConfig(options.executorConfig, this.suites); |
| 110 | const context = await this.setup(); |
| 111 | const artifactDir = options.sessionLog?.executorArtifactsDir; |
| 112 | if (!artifactDir) throw new InfrastructureError("gh-agent needs Vally's per-trial artifacts directory (run it with vally eval)"); |
| 113 | if (stimulus.turns?.length || stimulus.attachments?.length) { |
| 114 | throw new InfrastructureError("gh-agent supports single-turn prompts without attachments"); |
| 115 | } |
| 116 | await mkdir(artifactDir, { recursive: true, mode: 0o700 }); |
| 117 | const secrets = [context.token, context.actorToken ?? "", context.fixtureToken, context.modelToken].filter(Boolean); |
| 118 | const log = (text: string) => appendFileSync(join(artifactDir, "runtime.log"), redact(text, secrets), { mode: 0o600 }); |
| 119 | const skills = options.skills ?? []; |
| 120 | const label = `${stimulus.name} skill-${skills.length ? "on" : "off"}`; |
| 121 | const progress = (message: string) => { |
| 122 | const line = `[${new Date().toISOString().slice(11, 19)}] ${label}: ${message}`; |
| 123 | console.error(redact(line, secrets)); |
| 124 | log(line + "\n"); |
| 125 | }; |
| 126 | const home = await mkdtemp(join(tmpdir(), "gh-work-")); |
| 127 | const workDir = options.workDir; |
| 128 | const journalPath = join(artifactDir, "resources.json"); |
| 129 | const calls: Call[] = []; |
| 130 | const rawEvents: unknown[] = []; |
| 131 | let observerError: Error | undefined; |
| 132 | let contaminated: string | undefined; |
| 133 | const observer = new CommandObserver(calls, (call, phase) => { |
| 134 | const entry = phase === "output" && call.kind === "gh" |
| 135 | ? { type: phase, id: call.id, chunk: call.output.at(-1) } |
| 136 | : { type: phase, ...call }; |
| 137 | appendFileSync(join(artifactDir, "commands.jsonl"), redact(JSON.stringify(entry) + "\n", secrets), { mode: 0o600 }); |
| 138 | if (phase === "start" && call.kind === "gh") { |
| 139 | const command = `gh ${call.arguments.join(" ")}`; |
| 140 | progress(`agent ran: ${command.length > 140 ? `${command.slice(0, 137)}...` : command}`); |
| 141 | } |
| 142 | }); |
| 143 | let listening = false; |
| 144 | let inner: CopilotSdkExecutor | undefined; |
| 145 | let primary: unknown; |
| 146 | let driver: ActorDriver<unknown> | undefined; |
| 147 | try { |
| 148 | for (const hidden of [sourceRoot, artifactDir]) { |
| 149 | if (inside(await realpath(home), hidden) || inside(workDir, hidden)) { |
| 150 | throw new InfrastructureError("Agent directories must be outside the checkout and result directories"); |
| 151 | } |
| 152 | } |
| 153 | const variant = stimulusVariant(stimulus.tags?.variant, suite.variants); |
| 154 | const env = await prepareHome(home, context); |
| 155 | await observer.listen(join(home, ".commands.sock")); |
| 156 | listening = true; |
| 157 | const staged: Skill[] = []; |
| 158 | for (const skill of skills) { |
| 159 | if (!skill.path) throw new InfrastructureError(`Skill ${skill.name} has no path to stage`); |
| 160 | const target = join(home, ".skills", basename(dirname(skill.path))); |
| 161 | await cp(dirname(skill.path), target, { recursive: true, dereference: false }); |
| 162 | staged.push({ ...skill, path: join(target, basename(skill.path)) }); |
| 163 | } |
| 164 | const skillSource = process.env.GH_EVAL_SKILL_SOURCE?.trim(); |
| 165 | if (staged.length && skillSource) log(`gh skill: ${skillSource}\n`); |
no test coverage detected