Zero parameter grads.
(self)
| 3259 | self.optimizer.is_gradient_accumulation_boundary = is_boundary |
| 3260 | |
| 3261 | def zero_grad(self): |
| 3262 | """ |
| 3263 | Zero parameter grads. |
| 3264 | """ |
| 3265 | for param_name, param in self.module.named_parameters(): |
| 3266 | param.grad = None |
| 3267 | |
| 3268 | def clip_fp32_gradients(self): |
| 3269 | clip_grad_norm_(parameters=self.module.parameters(), max_norm=self.gradient_clipping(), mpu=self.mpu) |