MCPcopy Create free account
hub / github.com/cli/cli / trial

Method trial

evals/src/core/executor.ts:108–266  ·  view source on GitHub ↗
(stimulus: Stimulus, options: ExecutorOptions)

Source from the content-addressed store, hash-verified

106 }
107
108 private async trial(stimulus: Stimulus, options: ExecutorOptions): Promise<Trajectory> {
109 const { suite, scenario } = parseExecutorConfig(options.executorConfig, this.suites);
110 const context = await this.setup();
111 const artifactDir = options.sessionLog?.executorArtifactsDir;
112 if (!artifactDir) throw new InfrastructureError("gh-agent needs Vally's per-trial artifacts directory (run it with vally eval)");
113 if (stimulus.turns?.length || stimulus.attachments?.length) {
114 throw new InfrastructureError("gh-agent supports single-turn prompts without attachments");
115 }
116 await mkdir(artifactDir, { recursive: true, mode: 0o700 });
117 const secrets = [context.token, context.actorToken ?? "", context.fixtureToken, context.modelToken].filter(Boolean);
118 const log = (text: string) => appendFileSync(join(artifactDir, "runtime.log"), redact(text, secrets), { mode: 0o600 });
119 const skills = options.skills ?? [];
120 const label = `${stimulus.name} skill-${skills.length ? "on" : "off"}`;
121 const progress = (message: string) => {
122 const line = `[${new Date().toISOString().slice(11, 19)}] ${label}: ${message}`;
123 console.error(redact(line, secrets));
124 log(line + "\n");
125 };
126 const home = await mkdtemp(join(tmpdir(), "gh-work-"));
127 const workDir = options.workDir;
128 const journalPath = join(artifactDir, "resources.json");
129 const calls: Call[] = [];
130 const rawEvents: unknown[] = [];
131 let observerError: Error | undefined;
132 let contaminated: string | undefined;
133 const observer = new CommandObserver(calls, (call, phase) => {
134 const entry = phase === "output" && call.kind === "gh"
135 ? { type: phase, id: call.id, chunk: call.output.at(-1) }
136 : { type: phase, ...call };
137 appendFileSync(join(artifactDir, "commands.jsonl"), redact(JSON.stringify(entry) + "\n", secrets), { mode: 0o600 });
138 if (phase === "start" && call.kind === "gh") {
139 const command = `gh ${call.arguments.join(" ")}`;
140 progress(`agent ran: ${command.length > 140 ? `${command.slice(0, 137)}...` : command}`);
141 }
142 });
143 let listening = false;
144 let inner: CopilotSdkExecutor | undefined;
145 let primary: unknown;
146 let driver: ActorDriver<unknown> | undefined;
147 try {
148 for (const hidden of [sourceRoot, artifactDir]) {
149 if (inside(await realpath(home), hidden) || inside(workDir, hidden)) {
150 throw new InfrastructureError("Agent directories must be outside the checkout and result directories");
151 }
152 }
153 const variant = stimulusVariant(stimulus.tags?.variant, suite.variants);
154 const env = await prepareHome(home, context);
155 await observer.listen(join(home, ".commands.sock"));
156 listening = true;
157 const staged: Skill[] = [];
158 for (const skill of skills) {
159 if (!skill.path) throw new InfrastructureError(`Skill ${skill.name} has no path to stage`);
160 const target = join(home, ".skills", basename(dirname(skill.path)));
161 await cp(dirname(skill.path), target, { recursive: true, dereference: false });
162 staged.push({ ...skill, path: join(target, basename(skill.path)) });
163 }
164 const skillSource = process.env.GH_EVAL_SKILL_SOURCE?.trim();
165 if (staged.length && skillSource) log(`gh skill: ${skillSource}\n`);

Callers 1

executeMethod · 0.95

Calls 15

setupMethod · 0.95
listenMethod · 0.95
parseExecutorConfigFunction · 0.85
redactFunction · 0.85
insideFunction · 0.85
stimulusVariantFunction · 0.85
prepareHomeFunction · 0.85
effectiveAgentBudgetFunction · 0.85
saveFunction · 0.85
resolvePromptFunction · 0.85
builtinSkillNamesFunction · 0.85
contaminationFunction · 0.85

Tested by

no test coverage detected