Method auto_denan_recover

PATH/core/solvers/solver.py:384–395

(self)

        return False

    def auto_denan_recover(self):
        try:
            if self.C.rank == 0:
                self.logger.info('NaN or Inf encountered, recovering from {}\t'.format(self.last_save_iter))
            # recover model
            self.model.load_state_dict(self.last_state_dict, strict=True)
            # recover optimizer
            for g in self.optimizer.param_groups:
                for p in g['params']:
                    self.optimizer.state[p]['momentum_buffer'].copy_(self.last_optim_state_dict['state'][id(p)]['momentum_buffer'])
        except:
            raise RuntimeError('If NaN or Inf at iter 0, try lower lr. Otherwise please contact zhouyucong for a bug fix')

    def auto_denan_save(self):
        if self.last_save_iter < 100 or self.tmp.current_step - self.last_save_iter > 100: