Benchmark the model checkpoints.
(self)
| 441 | ) |
| 442 | |
| 443 | async def benchmark(self) -> bool: |
| 444 | """Benchmark the model checkpoints.""" |
| 445 | # benchmark on the latest checkpoint |
| 446 | if self.config.explorer.bench_on_latest_checkpoint: |
| 447 | self.explore_step_num = await self._checkpoint_weights_update() |
| 448 | await self.eval() |
| 449 | await self._finish_eval_step(prefix="bench") |
| 450 | return True |
| 451 | |
| 452 | # benchmark on base model |
| 453 | if self.config.explorer.eval_on_startup: |
| 454 | await self._finish_eval_step(prefix="bench") |
| 455 | |
| 456 | # benchmark on all checkpoints |
| 457 | all_ckp_steps = sorted( |
| 458 | [ |
| 459 | int(ckp.split("global_step_")[-1]) |
| 460 | for ckp in os.listdir(self.config.checkpoint_job_dir) |
| 461 | if os.path.isdir(os.path.join(self.config.checkpoint_job_dir, ckp)) |
| 462 | and ckp.startswith("global_step_") |
| 463 | ] |
| 464 | ) |
| 465 | for step_num in all_ckp_steps: |
| 466 | if step_num <= self.explore_step_num: |
| 467 | continue |
| 468 | self.explore_step_num = await self._checkpoint_weights_update(step_num=step_num) |
| 469 | await self.eval() |
| 470 | await self._finish_eval_step(prefix="bench") |
| 471 | return True |
| 472 | |
| 473 | async def save_checkpoint(self) -> None: |
| 474 | # save explore checkpoint |
nothing calls this directly
no test coverage detected