| 498 | torch.distributed.barrier() |
| 499 | |
| 500 | def eval_step(self, global_step, epoch): |
| 501 | cur_metric = self.eval() |
| 502 | cur_metric_str = f"cur metric, {', '.join(['{}: {}'.format(k, v) for k, v in cur_metric.items()])}" |
| 503 | self.logger.info(cur_metric_str) |
| 504 | |
| 505 | # logger metric |
| 506 | if self.writer is not None: |
| 507 | for k, v in cur_metric.items(): |
| 508 | if isinstance(v, (float, int)): |
| 509 | self.writer.add_scalar(f'EVAL/{k}', cur_metric[k], |
| 510 | global_step) |
| 511 | |
| 512 | if (cur_metric[self.eval_class.main_indicator] >= |
| 513 | self.best_metric[self.eval_class.main_indicator]): |
| 514 | self.best_metric.update(cur_metric) |
| 515 | self.best_metric['best_epoch'] = epoch |
| 516 | |
| 517 | if self.writer is not None: |
| 518 | self.writer.add_scalar( |
| 519 | f'EVAL/best_{self.eval_class.main_indicator}', |
| 520 | self.best_metric[self.eval_class.main_indicator], |
| 521 | global_step, |
| 522 | ) |
| 523 | |
| 524 | save_ckpt(self.model, |
| 525 | self.cfg, |
| 526 | self.optimizer, |
| 527 | self.lr_scheduler, |
| 528 | epoch, |
| 529 | global_step, |
| 530 | self.best_metric, |
| 531 | is_best=True, |
| 532 | prefix=None) |
| 533 | best_str = f"best metric, {', '.join(['{}: {}'.format(k, v) for k, v in self.best_metric.items()])}" |
| 534 | self.logger.info(best_str) |
| 535 | |
| 536 | def eval(self): |
| 537 | self.model.eval() |