| 408 | self.last_save_iter = self.tmp.current_step |
| 409 | |
| 410 | def gather_result(self): |
| 411 | tmp = self.tmp |
| 412 | ginfo = self.ginfo |
| 413 | |
| 414 | vreduce(tmp.vloss, tmp.raw_loss.data, group=ginfo.group) |
| 415 | vreduce(tmp.vtop1, tmp.top1, group=ginfo.group) |
| 416 | |
| 417 | vgather(tmp.loss_list, tmp.vloss.avg) |
| 418 | vgather(tmp.top1_list, tmp.vtop1.avg) |
| 419 | |
| 420 | if self.auto_clip: |
| 421 | vreduce(tmp.vbackbone_grad_norm, torch.Tensor([tmp.backbone_grad_norm/ginfo.task_size]).cuda(), group=ginfo.group) |
| 422 | vgather(tmp.backbone_grad_norm_list, tmp.vbackbone_grad_norm.avg) |
| 423 | vreduce(tmp.vneck_grad_norm, torch.Tensor([tmp.neck_grad_norm/ginfo.task_size]).cuda(), group=ginfo.group) |
| 424 | vgather(tmp.neck_grad_norm_list, tmp.vneck_grad_norm.avg) |
| 425 | vreduce(tmp.vdecoder_grad_norm, torch.Tensor([tmp.decoder_grad_norm/ginfo.task_size]).cuda(), group=ginfo.group) |
| 426 | vgather(tmp.decoder_grad_norm_list, tmp.vdecoder_grad_norm.avg) |
| 427 | |
| 428 | vreduce(tmp.vbackbone_grad_thresh, torch.Tensor([tmp.backbone_grad_thresh/ginfo.task_size]).cuda(), group=ginfo.group) |
| 429 | vgather(tmp.backbone_grad_thresh_list, tmp.vbackbone_grad_thresh.avg) |
| 430 | vreduce(tmp.vneck_grad_thresh, torch.Tensor([tmp.neck_grad_thresh/ginfo.task_size]).cuda(), group=ginfo.group) |
| 431 | vgather(tmp.neck_grad_thresh_list, tmp.vneck_grad_thresh.avg) |
| 432 | vreduce(tmp.vdecoder_grad_thresh, torch.Tensor([tmp.decoder_grad_thresh/ginfo.task_size]).cuda(), group=ginfo.group) |
| 433 | vgather(tmp.decoder_grad_thresh_list, tmp.vdecoder_grad_thresh.avg) |
| 434 | |
| 435 | elif self.manual_clip: |
| 436 | if self.clip_grad_backbone > 0: |
| 437 | vreduce(tmp.vbackbone_grad_norm, torch.Tensor([tmp.backbone_grad_norm/ginfo.task_size]).cuda(), group=ginfo.group) |
| 438 | vgather(tmp.backbone_grad_norm_list, tmp.vbackbone_grad_norm.avg) |
| 439 | if self.clip_grad_neck > 0: |
| 440 | vreduce(tmp.vneck_grad_norm, torch.Tensor([tmp.neck_grad_norm/ginfo.task_size]).cuda(), group=ginfo.group) |
| 441 | vgather(tmp.neck_grad_norm_list, tmp.vneck_grad_norm.avg) |
| 442 | if self.clip_grad_decoder > 0: |
| 443 | vreduce(tmp.vdecoder_grad_norm, torch.Tensor([tmp.decoder_grad_norm/ginfo.task_size]).cuda(), group=ginfo.group) |
| 444 | vgather(tmp.decoder_grad_norm_list, tmp.vdecoder_grad_norm.avg) |
| 445 | |
| 446 | def play_with_grads(self): |
| 447 | if self.clip_grad > 0: |