| 338 | loss.backward() |
| 339 | |
| 340 | def grad_norm(self, norm_type): |
| 341 | results = {} |
| 342 | total_norm = 0 |
| 343 | for name, p in self.named_parameters(): |
| 344 | if p.requires_grad: |
| 345 | try: |
| 346 | param_norm = p.grad.data.norm(norm_type) |
| 347 | total_norm += param_norm ** norm_type |
| 348 | norm = param_norm ** (1 / norm_type) |
| 349 | |
| 350 | grad = round(norm.data.cpu().numpy().flatten()[0], 3) |
| 351 | results['grad_{}_norm_{}'.format(norm_type, name)] = grad |
| 352 | except Exception: |
| 353 | # this param had no grad |
| 354 | pass |
| 355 | |
| 356 | total_norm = total_norm ** (1. / norm_type) |
| 357 | grad = round(total_norm.data.cpu().numpy().flatten()[0], 3) |
| 358 | results['grad_{}_norm_total'.format(norm_type)] = grad |
| 359 | return results |