(self)
| 341 | self._last_gs = -1 |
| 342 | |
| 343 | def _before_train(self): |
| 344 | stats = JSONWriter.load_existing_json() |
| 345 | self._fname = os.path.join(logger.get_logger_dir(), JSONWriter.FILENAME) |
| 346 | if stats is not None: |
| 347 | try: |
| 348 | epoch = stats[-1]['epoch_num'] + 1 |
| 349 | except Exception: |
| 350 | epoch = None |
| 351 | |
| 352 | # check against the current training settings |
| 353 | # therefore this logic needs to be in before_train stage |
| 354 | starting_epoch = self.trainer.loop.starting_epoch |
| 355 | if epoch is None or epoch == starting_epoch: |
| 356 | logger.info("Found existing JSON inside {}, will append to it.".format(logger.get_logger_dir())) |
| 357 | self._stats = stats |
| 358 | else: |
| 359 | logger.warn( |
| 360 | "History epoch={} from JSON is not the predecessor of the current starting_epoch={}".format( |
| 361 | epoch - 1, starting_epoch)) |
| 362 | logger.warn("If you want to resume old training, either use `AutoResumeTrainConfig` " |
| 363 | "or correctly set the new starting_epoch yourself to avoid inconsistency. ") |
| 364 | |
| 365 | backup_fname = JSONWriter.FILENAME + '.' + datetime.now().strftime('%m%d-%H%M%S') |
| 366 | backup_fname = os.path.join(logger.get_logger_dir(), backup_fname) |
| 367 | |
| 368 | logger.warn("Now, we will train with starting_epoch={} and backup old json to {}".format( |
| 369 | self.trainer.loop.starting_epoch, backup_fname)) |
| 370 | shutil.move(self._fname, backup_fname) |
| 371 | |
| 372 | # in case we have something to log here. |
| 373 | self._trigger() |
| 374 | |
| 375 | def _trigger_step(self): |
| 376 | # will do this in trigger_epoch |
nothing calls this directly
no test coverage detected