(args, scenario, fixture, log_dir, eval_env)
| 369 | |
| 370 | |
| 371 | def drive_hermes(args, scenario, fixture, log_dir, eval_env): |
| 372 | profile_dir = ensure_hermes_profile( |
| 373 | args.hermes_profile, args.model, args.provider, args.tracedecay_bin, fixture, eval_env |
| 374 | ) |
| 375 | max_turns = args.max_turns or scenario["real_model"].get("max_turns", 8) |
| 376 | env = dict(eval_env) |
| 377 | env["HERMES_HOME"] = str(profile_dir) |
| 378 | provider_env_passthrough(env) |
| 379 | transcripts = [] |
| 380 | for index, prompt in enumerate(scenario["real_model"]["prompts"], start=1): |
| 381 | cmd = [ |
| 382 | "uv", |
| 383 | "run", |
| 384 | "python", |
| 385 | "cli.py", |
| 386 | "-q", |
| 387 | prompt, |
| 388 | "--model", |
| 389 | args.model, |
| 390 | "--max-turns", |
| 391 | str(max_turns), |
| 392 | ] |
| 393 | if args.provider: |
| 394 | cmd += ["--provider", args.provider] |
| 395 | started = datetime.datetime.now(datetime.timezone.utc) |
| 396 | result = run(cmd, cwd=args.hermes_dir, env=env, timeout=900, check=False) |
| 397 | elapsed = (datetime.datetime.now(datetime.timezone.utc) - started).total_seconds() |
| 398 | log_path = log_dir / f"{scenario['id']}-prompt{index}.log" |
| 399 | log_path.write_text(result.stdout) |
| 400 | transcripts.append( |
| 401 | { |
| 402 | "prompt": prompt, |
| 403 | "exit_code": result.returncode, |
| 404 | "seconds": round(elapsed, 1), |
| 405 | "log": str(log_path.relative_to(RUNS_DIR.parent)), |
| 406 | "turn_valid": turn_is_valid(result), |
| 407 | "usage": read_hermes_usage(profile_dir, started.timestamp()), |
| 408 | "token_hints": extract_token_hints(result.stdout), |
| 409 | } |
| 410 | ) |
| 411 | return transcripts |
| 412 | |
| 413 | |
| 414 | def read_hermes_usage(profile_dir, started_after): |
no test coverage detected