Function run_eval

contributing/samples/integrations/gepa/experiment.py:596–639

Runs evaluation on the test set using the given instructions. Args: output_dir: The directory to save evaluation results. instructions: The system instructions to evaluate. config: The experiment configuration.

(output_dir: str, instructions: str, config: ExperimentConfig)

def run_eval(output_dir: str, instructions: str, config: ExperimentConfig):
  """Runs evaluation on the test set using the given instructions.

  Args:
    output_dir: The directory to save evaluation results.
    instructions: The system instructions to evaluate.
    config: The experiment configuration.
  """
  eval_dataset = _get_dataset(config.eval_dataset)
  tau_bench_run_config = RunConfig(
      env=config.tau_bench_env,
      model=config.agent_model,
      model_provider=config.agent_model_provider,
      user_model=config.user_model,
      user_model_provider=config.user_model_provider,
      agent_strategy='tool-calling',
      user_strategy='llm',
      max_concurrency=config.max_concurrency,
      num_trials=config.num_eval_trials,
      task_ids=eval_dataset,
      log_dir=output_dir,
      task_split=config.eval_dataset.split,
  )
  with open(os.path.join(output_dir, 'prompt.txt'), 'w') as f:
    f.write(instructions)

  json.dump(
      tau_bench_run_config.model_dump(),
      open(os.path.join(output_dir, 'run_config.json'), 'w'),
  )
  tau_bench_results = run_tau_bench_rollouts(
      tau_bench_run_config,
      system_instruction=instructions,
      rater=_rater(config) if config.use_rater else None,
  )
  total = len(tau_bench_results)
  numerator = sum(1 for res in tau_bench_results if res.reward == 1)
  print(
      f'average reward (total={total}): {numerator/total if total > 0 else 0}'
  )
  json.dump(
      dict(results=[r.model_dump() for r in tau_bench_results]),
      open(os.path.join(output_dir, 'results.json'), 'w'),
  )