| 487 | running_state.restart_runner_on_cancel = restart_runners |
| 488 | |
| 489 | async def start(self) -> None: |
| 490 | if self.running: |
| 491 | return |
| 492 | self.running = True |
| 493 | # bounded-concurrency runner prepare with in-slot retry (avoids glibc getenv races) |
| 494 | _sem = asyncio.Semaphore(max(1, self.config.explorer.runner_prepare_concurrency)) |
| 495 | _retries = max(0, self.config.explorer.runner_prepare_max_retries) |
| 496 | |
| 497 | async def _create_limited(i: int) -> None: |
| 498 | async with _sem: |
| 499 | for attempt in range(_retries + 1): |
| 500 | try: |
| 501 | await self._create_runner(i) |
| 502 | return |
| 503 | except Exception as e: |
| 504 | if attempt == _retries: |
| 505 | raise |
| 506 | self.logger.warning( |
| 507 | f"Runner {i} prepare failed ({attempt + 1}/{_retries + 1}), retrying: {e}" |
| 508 | ) |
| 509 | await asyncio.sleep(2.0 * (attempt + 1)) |
| 510 | |
| 511 | await asyncio.gather(*[_create_limited(i) for i in range(self.runner_num)]) |
| 512 | self.scheduler_task = asyncio.create_task(self._scheduler_loop()) |
| 513 | ready_refs = [runner.runner.__ray_ready__.remote() for runner in self.runners.values()] |
| 514 | await asyncio.gather(*ready_refs) |
| 515 | self.monitor_task = asyncio.create_task(self._monitor_runner_state_loop()) |
| 516 | self.logger.info(f"Starting Scheduler with {self.runner_num} runners") |
| 517 | |
| 518 | async def stop(self) -> None: |
| 519 | if not self.running: |