(self)
| 303 | self.last_iter = checkpoint['step'] - 1 |
| 304 | |
| 305 | def pre_run(self): |
| 306 | tmp = self.tmp |
| 307 | tmp.vtask_time = AverageMeter(10) |
| 308 | tmp.vbatch_time = AverageMeter(10) |
| 309 | tmp.vdata_time = AverageMeter(10) |
| 310 | tmp.vloss = AverageMeter(10) |
| 311 | tmp.vtop1 = AverageMeter(10) |
| 312 | |
| 313 | printlog(f">>> sanity check: attempting torch.Tensor(1).cuda(), check task_sp_list if stuck") |
| 314 | torch.Tensor(1).cuda() |
| 315 | printlog(f">>> sanity check: torch.Tensor(1).cuda() passed") |
| 316 | |
| 317 | tmp.loss_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 318 | tmp.top1_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 319 | |
| 320 | tmp.vbackbone_grad_norm = AverageMeter(10) |
| 321 | tmp.backbone_grad_norm_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 322 | tmp.vneck_grad_norm = AverageMeter(10) |
| 323 | tmp.neck_grad_norm_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 324 | tmp.vdecoder_grad_norm = AverageMeter(10) |
| 325 | tmp.decoder_grad_norm_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 326 | |
| 327 | tmp.vbackbone_grad_thresh = AverageMeter(10) |
| 328 | tmp.backbone_grad_thresh_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 329 | tmp.vneck_grad_thresh = AverageMeter(10) |
| 330 | tmp.neck_grad_thresh_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 331 | tmp.vdecoder_grad_thresh = AverageMeter(10) |
| 332 | tmp.decoder_grad_thresh_list = [torch.Tensor(1).cuda() for _ in range(self.C.world_size)] |
| 333 | |
| 334 | self.model.train() |
| 335 | |
| 336 | def gather_result(self): |
| 337 | tmp = self.tmp |
no test coverage detected