(spec: ToolSpec, sessions: int, cwd: Path, timeout_s: float, settle_s: float)
| 346 | |
| 347 | |
| 348 | def run_tool(spec: ToolSpec, sessions: int, cwd: Path, timeout_s: float, settle_s: float) -> ToolRunResult: |
| 349 | notes: list[str] = [] |
| 350 | version = version_for(spec) |
| 351 | launches: list[SessionLaunch] = [] |
| 352 | cleanup_pgids: list[int] = [] |
| 353 | temp_root: str | None = None |
| 354 | try: |
| 355 | if spec.jcode: |
| 356 | temp_root = tempfile.mkdtemp(prefix="jcode-memory-bench-") |
| 357 | env = os.environ.copy() |
| 358 | if spec.env: |
| 359 | env.update(spec.env) |
| 360 | env["JCODE_HOME"] = os.path.join(temp_root, "home") |
| 361 | env["JCODE_RUNTIME_DIR"] = os.path.join(temp_root, "run") |
| 362 | env["JCODE_TEMP_SERVER"] = "1" |
| 363 | env["JCODE_SERVER_OWNER_PID"] = str(os.getpid()) |
| 364 | os.makedirs(env["JCODE_HOME"], exist_ok=True) |
| 365 | os.makedirs(env["JCODE_RUNTIME_DIR"], exist_ok=True) |
| 366 | for auth_name in ( |
| 367 | "anthropic-auth.json", |
| 368 | "openai-auth.json", |
| 369 | "antigravity_oauth.json", |
| 370 | "gemini_oauth.json", |
| 371 | "config.toml", |
| 372 | ): |
| 373 | real_auth = Path.home() / ".jcode" / auth_name |
| 374 | bench_auth = Path(env["JCODE_HOME"]) / auth_name |
| 375 | if real_auth.exists() and not bench_auth.exists(): |
| 376 | bench_auth.symlink_to(real_auth) |
| 377 | if spec.name == "jcode_memory_on": |
| 378 | real_models = Path.home() / ".jcode" / "models" |
| 379 | bench_models = Path(env["JCODE_HOME"]) / "models" |
| 380 | if real_models.exists() and not bench_models.exists(): |
| 381 | bench_models.symlink_to(real_models) |
| 382 | socket_path = os.path.join(env["JCODE_RUNTIME_DIR"], "bench.sock") |
| 383 | server_proc = subprocess.Popen( |
| 384 | [spec.argv[0], "--no-update", "--no-selfdev", "serve", "--socket", socket_path], |
| 385 | cwd=str(cwd), |
| 386 | env=env, |
| 387 | stdin=subprocess.DEVNULL, |
| 388 | stdout=subprocess.DEVNULL, |
| 389 | stderr=subprocess.DEVNULL, |
| 390 | preexec_fn=os.setsid, |
| 391 | ) |
| 392 | cleanup_pgids.append(os.getpgid(server_proc.pid)) |
| 393 | if not wait_for_socket(socket_path, timeout_s): |
| 394 | raise RuntimeError("jcode server did not become ready") |
| 395 | if spec.name == "jcode_memory_on": |
| 396 | time.sleep(max(settle_s, 5.0)) |
| 397 | per_session_settle = max(settle_s, 2.0) if spec.name == "jcode_memory_on" else settle_s |
| 398 | for _ in range(sessions): |
| 399 | launches.append( |
| 400 | launch_interactive( |
| 401 | [spec.argv[0], "--no-update", "--no-selfdev", "--socket", socket_path], |
| 402 | cwd, |
| 403 | env, |
| 404 | timeout_s, |
| 405 | per_session_settle, |
no test coverage detected