| 282 | |
| 283 | |
| 284 | class LatestModelCheckpoint(ModelCheckpoint): |
| 285 | def __init__(self, filepath, monitor='val_loss', verbose=0, num_ckpt_keep=5, |
| 286 | save_weights_only=False, mode='auto', period=1, prefix='model', save_best=True): |
| 287 | super(ModelCheckpoint, self).__init__() |
| 288 | self.monitor = monitor |
| 289 | self.verbose = verbose |
| 290 | self.filepath = filepath |
| 291 | os.makedirs(filepath, exist_ok=True) |
| 292 | self.num_ckpt_keep = num_ckpt_keep |
| 293 | self.save_best = save_best |
| 294 | self.save_weights_only = save_weights_only |
| 295 | self.period = period |
| 296 | self.epochs_since_last_check = 0 |
| 297 | self.prefix = prefix |
| 298 | self.best_k_models = {} |
| 299 | # {filename: monitor} |
| 300 | self.kth_best_model = '' |
| 301 | self.save_top_k = 1 |
| 302 | self.task = None |
| 303 | if mode == 'min': |
| 304 | self.monitor_op = np.less |
| 305 | self.best = np.Inf |
| 306 | self.mode = 'min' |
| 307 | elif mode == 'max': |
| 308 | self.monitor_op = np.greater |
| 309 | self.best = -np.Inf |
| 310 | self.mode = 'max' |
| 311 | else: |
| 312 | if 'acc' in self.monitor or self.monitor.startswith('fmeasure'): |
| 313 | self.monitor_op = np.greater |
| 314 | self.best = -np.Inf |
| 315 | self.mode = 'max' |
| 316 | else: |
| 317 | self.monitor_op = np.less |
| 318 | self.best = np.Inf |
| 319 | self.mode = 'min' |
| 320 | if os.path.exists(f'{self.filepath}/best_valid.npy'): |
| 321 | self.best = np.load(f'{self.filepath}/best_valid.npy')[0] |
| 322 | |
| 323 | def get_all_ckpts(self): |
| 324 | return sorted(glob.glob(f'{self.filepath}/{self.prefix}_ckpt_steps_*.ckpt'), |
| 325 | key=lambda x: -int(re.findall('.*steps\_(\d+)\.ckpt', x)[0])) |
| 326 | |
| 327 | def on_epoch_end(self, epoch, logs=None): |
| 328 | logs = logs or {} |
| 329 | self.epochs_since_last_check += 1 |
| 330 | best_filepath = f'{self.filepath}/{self.prefix}_ckpt_best.pt' |
| 331 | if self.epochs_since_last_check >= self.period: |
| 332 | self.epochs_since_last_check = 0 |
| 333 | filepath = f'{self.filepath}/{self.prefix}_ckpt_steps_{self.task.global_step}.ckpt' |
| 334 | if self.verbose > 0: |
| 335 | logging.info(f'Epoch {epoch:05d}@{self.task.global_step}: saving model to {filepath}') |
| 336 | self._save_model(filepath) |
| 337 | for old_ckpt in self.get_all_ckpts()[self.num_ckpt_keep:]: |
| 338 | subprocess.check_call(f'rm -rf "{old_ckpt}"', shell=True) |
| 339 | if self.verbose > 0: |
| 340 | logging.info(f'Delete ckpt: {os.path.basename(old_ckpt)}') |
| 341 | current = logs.get(self.monitor) |