Evaluation on all evaluation data samples.
(self)
| 405 | return self.explore_step_num % self.config.explorer.eval_interval == 0 |
| 406 | |
| 407 | async def eval(self): |
| 408 | """Evaluation on all evaluation data samples.""" |
| 409 | if len(self.config.buffer.explorer_input.eval_tasksets) == 0: |
| 410 | self.logger.info("No evaluation data samples. Skip evaluation.") |
| 411 | return |
| 412 | |
| 413 | self.eval_start_time = time.time() |
| 414 | self.logger.info(f"Evaluation at step {self.explore_step_num} started.") |
| 415 | |
| 416 | if self.config.buffer.explorer_input.default_eval_workflow_type: |
| 417 | self.logger.info( |
| 418 | f"Use '{self.config.buffer.explorer_input.default_eval_workflow_type}' for evaluation." |
| 419 | ) |
| 420 | |
| 421 | for eval_taskset_config in self.config.buffer.explorer_input.eval_tasksets: |
| 422 | self.logger.info( |
| 423 | f"Evaluation on {eval_taskset_config.name} at step {self.explore_step_num} started." |
| 424 | ) |
| 425 | eval_taskset = get_buffer_reader(eval_taskset_config) |
| 426 | eval_batch_id = f"{self.explore_step_num}/{eval_taskset_config.name}" |
| 427 | self.pending_eval_tasks.append((self.explore_step_num, eval_taskset_config.name)) |
| 428 | eval_tasks = [] |
| 429 | while True: |
| 430 | try: |
| 431 | eval_tasks.extend(await eval_taskset.read()) |
| 432 | except StopAsyncIteration: |
| 433 | break |
| 434 | assert ( |
| 435 | self.rollout_coordinator is not None |
| 436 | ), "Rollout coordinator must be prepared first." |
| 437 | await self.rollout_coordinator.submit_batch.remote( |
| 438 | batch_id=eval_batch_id, |
| 439 | tasks=eval_tasks, |
| 440 | batch_type="eval", |
| 441 | ) |
| 442 | |
| 443 | async def benchmark(self) -> bool: |
| 444 | """Benchmark the model checkpoints.""" |
no test coverage detected