Evaluate model.
(config: Config, *, timeout: Optional[float] = None)
| 57 | |
| 58 | |
| 59 | def bench(config: Config, *, timeout: Optional[float] = None) -> StageStatus: |
| 60 | """Evaluate model.""" |
| 61 | from trinity.explorer.explorer import Explorer |
| 62 | |
| 63 | config.explorer.name = "benchmark" |
| 64 | explorer = Explorer.get_actor(config) |
| 65 | startup_started_at = time.perf_counter() |
| 66 | startup_time_sec: Optional[float] = None |
| 67 | try: |
| 68 | ray.get(explorer.prepare.remote(), timeout=timeout) |
| 69 | startup_time_sec = time.perf_counter() - startup_started_at |
| 70 | |
| 71 | run_started_at = time.perf_counter() |
| 72 | ray.get(explorer.benchmark.remote(), timeout=timeout) |
| 73 | execution_time_sec = time.perf_counter() - run_started_at |
| 74 | logger.info("Benchmark finished.") |
| 75 | return StageStatus( |
| 76 | stage="bench", |
| 77 | success=True, |
| 78 | startup_time_sec=startup_time_sec, |
| 79 | execution_time_sec=execution_time_sec, |
| 80 | total_time_sec=time.perf_counter() - startup_started_at, |
| 81 | ) |
| 82 | except Exception as exc: |
| 83 | error = _build_stage_error(exc) |
| 84 | logger.error(f"Benchmark failed:\n{error.traceback_text}") |
| 85 | return StageStatus( |
| 86 | stage="bench", |
| 87 | success=False, |
| 88 | startup_time_sec=startup_time_sec, |
| 89 | execution_time_sec=None, |
| 90 | total_time_sec=time.perf_counter() - startup_started_at, |
| 91 | error=error, |
| 92 | ) |
| 93 | finally: |
| 94 | ray.get(explorer.shutdown.remote(), timeout=timeout) |
| 95 | |
| 96 | |
| 97 | def explore(config: Config, *, timeout: Optional[float] = None) -> StageStatus: |