Function runOnce

src/eval.ts:41–166

(
    agentName: string,
    modelId: string,
    taskId: string,
    opts: {
      logger: Logger.Instance;
    },
  )

  }

  async function runOnce(
    agentName: string,
    modelId: string,
    taskId: string,
    opts: {
      logger: Logger.Instance;
    },
  ) {
    const agent = Agent.get(agentName);
    Agent.validateModel(agent, modelId);
    const task = await Task.get(taskId);
    const cwd = await mkdtemp(join(tmpdir(), "openreval-"));
    $.cwd(cwd);

    try {
      opts.logger.log(`Cloning repository to ${cwd}...`);
      await cloneRepositoryAtCommit(task.source.repo, task.source.from);

      opts.logger.log(`Running pre-task commands...`);
      const beforeResults: Record<string, Metric.CommandExecution[]> = {};
      for (const { name, args } of task.metrics) {
        if (!args) continue;
        const cl = opts.logger.child(`[criterion ${name}]`);
        await runCommands(args.setup, { logger: cl, cwd });
        const results = await runCommands(args.commands, { logger: cl, cwd });
        beforeResults[name] = results;
      }

      opts.logger.log(`Running task...`);
      let duration = 0;
      const usage = { input: 0, output: 0, cost: 0 };
      const actions: string[] = [];
      for (const { commit, prompt } of task.prompts) {
        const cl = opts.logger.child(
          `[prompt ${task.source.repo.split("/")[1]}@${commit.slice(0, 7)}]`,
        );

        const startedAt = Date.now();
        const result = await agent.definition.run(modelId, prompt, {
          cwd,
          logger: cl,
        });
        duration += Date.now() - startedAt;

        // Only accumulate usage from the successful result
        usage.input += result.usage.input;
        usage.output += result.usage.output;
        usage.cost += result.usage.cost;

        // Collect actions from this task
        actions.push(...result.actions);
      }

      opts.logger.log(`Scoring...`);
      await finalizeChanges(task.source.from);
      const diff = await generateDiff(task.source.from);
      const allScores = [];
      for (const { name, weight, args } of task.metrics) {