Experimental: drives `cursor-agent -p` against the fixture's local MCP setup.
(args, scenario, fixture, log_dir, eval_env)
| 444 | |
| 445 | |
| 446 | def drive_cursor_agent(args, scenario, fixture, log_dir, eval_env): |
| 447 | """Experimental: drives `cursor-agent -p` against the fixture's local MCP setup.""" |
| 448 | run( |
| 449 | [args.tracedecay_bin, "install", "--agent", "cursor", "--local"], |
| 450 | cwd=fixture, |
| 451 | env=eval_env, |
| 452 | timeout=120, |
| 453 | ) |
| 454 | transcripts = [] |
| 455 | for index, prompt in enumerate(scenario["real_model"]["prompts"], start=1): |
| 456 | cmd = ["cursor-agent", "-p", "--output-format", "text", "--model", args.model, prompt] |
| 457 | started = datetime.datetime.now(datetime.timezone.utc) |
| 458 | result = run(cmd, cwd=fixture, env=eval_env, timeout=900, check=False) |
| 459 | elapsed = (datetime.datetime.now(datetime.timezone.utc) - started).total_seconds() |
| 460 | log_path = log_dir / f"{scenario['id']}-prompt{index}.log" |
| 461 | log_path.write_text(result.stdout) |
| 462 | transcripts.append( |
| 463 | { |
| 464 | "prompt": prompt, |
| 465 | "exit_code": result.returncode, |
| 466 | "seconds": round(elapsed, 1), |
| 467 | "log": str(log_path.relative_to(RUNS_DIR.parent)), |
| 468 | "turn_valid": turn_is_valid(result), |
| 469 | "usage": None, |
| 470 | "token_hints": extract_token_hints(result.stdout), |
| 471 | } |
| 472 | ) |
| 473 | return transcripts |
| 474 | |
| 475 | |
| 476 | def turn_is_valid(result): |
no test coverage detected