(self)
| 441 | self.tmp.input_var[k] = v |
| 442 | |
| 443 | def forward(self): |
| 444 | ## set random seed with current_step at each iteration |
| 445 | try: |
| 446 | self._set_randomseed(self.randomseed_pool[self.tmp.current_step]) |
| 447 | except: # workaround for reid task resumed sampler/loader bug damaging newest_checkpoints at the end of training |
| 448 | time.sleep(60) |
| 449 | raise ValueError(f"max_iter: {self.config.max_iter} current_step(-1): {self.tmp.current_step} " |
| 450 | f"rank: {self.C.rank}, task_id: " |
| 451 | f"{self.ginfo.task_id} (<--- I guess its reid task) task_rank: {self.ginfo.task_rank}" |
| 452 | f"This error is a reminder that we caught a data_loader length bug (should be from reid " |
| 453 | f"task), but the program should end normally with final checkpoint intact") |
| 454 | |
| 455 | tmp = self.tmp |
| 456 | ginfo = self.ginfo |
| 457 | |
| 458 | oom = False |
| 459 | try: |
| 460 | output = self.model(tmp.input_var, tmp.current_step) |
| 461 | except RuntimeError as mem_error: |
| 462 | printlog(f"*****\n" |
| 463 | f"***** encountered potential mem_error, current node: " |
| 464 | f"{os.environ['SLURM_NODEID']} - {os.environ['SLURMD_NODENAME']}" |
| 465 | f"task_id: {self.ginfo.task_id}" |
| 466 | f"\n*****") |
| 467 | printlog(f"error_message:\n{mem_error}") |
| 468 | printlog(traceback.format_exc()) |
| 469 | oom = True |
| 470 | if oom: |
| 471 | # python exception object holds a reference to the stack frame where the error was raised, which |
| 472 | # prevents the original tensor objects from being freed torch.cuda.empty_cache() |
| 473 | torch.cuda.empty_cache() |
| 474 | try: |
| 475 | output = self.model(tmp.input_var, tmp.current_step) |
| 476 | except RuntimeError as mem_error: |
| 477 | printlog(f"*****\n" |
| 478 | f"***** encountered potential mem_error, **restart attempt failed** current node: " |
| 479 | f"{os.environ['SLURM_NODEID']} - {os.environ['SLURMD_NODENAME']}" |
| 480 | f"\n*****") |
| 481 | raise mem_error |
| 482 | |
| 483 | tmp.output = output |
| 484 | tmp.raw_losses = output['loss'] # TODO: log all losses separately |
| 485 | if isinstance(tmp.raw_losses, dict): # only key with loss are used for loss computation |
| 486 | tmp.raw_loss = sum(tmp.raw_losses[k] for k in tmp.raw_losses.keys() if 'loss' in k) / ginfo.task_size |
| 487 | else: |
| 488 | tmp.raw_loss = tmp.raw_losses / ginfo.task_size |
| 489 | tmp.raw_losses = {"total_loss": tmp.raw_losses} |
| 490 | |
| 491 | if 'top1' in output: |
| 492 | tmp.raw_top1 = output['top1'] / ginfo.task_size |
| 493 | else: |
| 494 | tmp.raw_top1 = torch.zeros(1).cuda() |
| 495 | tmp.loss = tmp.raw_loss * ginfo.task_weight |
| 496 | tmp.top1 = tmp.raw_top1 |
| 497 | |
| 498 | def backward(self): |
| 499 | self.optimizer.zero_grad() |
no test coverage detected