(opt, model, max_grad_norm=2.0)
| 122 | |
| 123 | |
| 124 | def grad_clip(opt, model, max_grad_norm=2.0): |
| 125 | if hasattr(opt, "clip_grad_norm"): |
| 126 | # Some optimizers (like the sharded optimizer) have a specific way to do gradient clipping |
| 127 | opt.clip_grad_norm(max_grad_norm) |
| 128 | else: |
| 129 | # Revert to normal clipping otherwise, handling Apex or full precision |
| 130 | torch.nn.utils.clip_grad_norm_( |
| 131 | model.parameters(), # amp.master_params(self.opt) if self.use_apex else |
| 132 | max_grad_norm, |
| 133 | ) |
| 134 | |
| 135 | |
| 136 | def get_latest_checkpoint(checkpoint_dir): |