(self, epoch, logs=None)
| 325 | key=lambda x: -int(re.findall('.*steps\_(\d+)\.ckpt', x)[0])) |
| 326 | |
| 327 | def on_epoch_end(self, epoch, logs=None): |
| 328 | logs = logs or {} |
| 329 | self.epochs_since_last_check += 1 |
| 330 | best_filepath = f'{self.filepath}/{self.prefix}_ckpt_best.pt' |
| 331 | if self.epochs_since_last_check >= self.period: |
| 332 | self.epochs_since_last_check = 0 |
| 333 | filepath = f'{self.filepath}/{self.prefix}_ckpt_steps_{self.task.global_step}.ckpt' |
| 334 | if self.verbose > 0: |
| 335 | logging.info(f'Epoch {epoch:05d}@{self.task.global_step}: saving model to {filepath}') |
| 336 | self._save_model(filepath) |
| 337 | for old_ckpt in self.get_all_ckpts()[self.num_ckpt_keep:]: |
| 338 | subprocess.check_call(f'rm -rf "{old_ckpt}"', shell=True) |
| 339 | if self.verbose > 0: |
| 340 | logging.info(f'Delete ckpt: {os.path.basename(old_ckpt)}') |
| 341 | current = logs.get(self.monitor) |
| 342 | if current is not None and self.save_best: |
| 343 | if self.monitor_op(current, self.best): |
| 344 | self.best = current |
| 345 | if self.verbose > 0: |
| 346 | logging.info( |
| 347 | f'Epoch {epoch:05d}@{self.task.global_step}: {self.monitor} reached' |
| 348 | f' {current:0.5f} (best {self.best:0.5f}), saving model to' |
| 349 | f' {best_filepath} as top 1') |
| 350 | self._save_model(best_filepath) |
| 351 | np.save(f'{self.filepath}/best_valid.npy', [self.best]) |
| 352 | |
| 353 | |
| 354 | class BaseTrainer: |
no test coverage detected