Zero parameter grads.
(self)
| 3156 | self.optimizer.is_gradient_accumulation_boundary = is_boundary |
| 3157 | |
| 3158 | def zero_grad(self): |
| 3159 | """ |
| 3160 | Zero parameter grads. |
| 3161 | """ |
| 3162 | for param_name, param in self.module.named_parameters(): |
| 3163 | param.grad = None |
| 3164 | |
| 3165 | def clip_fp32_gradients(self): |
| 3166 | clip_grad_norm_(parameters=self.module.parameters(), max_norm=self.gradient_clipping(), mpu=self.mpu) |