(self)
| 365 | return False |
| 366 | |
| 367 | def auto_denan_check(self): |
| 368 | tmp = self.tmp |
| 369 | ginfo = self.ginfo |
| 370 | |
| 371 | drop_flag = 0 |
| 372 | if np.isnan(tmp.loss.data.item()) or np.isinf(tmp.loss.data.item()): |
| 373 | drop_flag = 1 |
| 374 | |
| 375 | drop_flag = torch.Tensor([drop_flag]).cuda() |
| 376 | dist.allreduce(drop_flag) |
| 377 | |
| 378 | drop_flag = drop_flag.item() |
| 379 | if drop_flag > 0: |
| 380 | return True |
| 381 | |
| 382 | return False |
| 383 | |
| 384 | def auto_denan_recover(self): |
| 385 | try: |