Creates gradient clipping closure to clip by value or by norm, according to the provided config.
(cfg: CfgNode)
| 18 | |
| 19 | |
| 20 | def _create_gradient_clipper(cfg: CfgNode) -> _GradientClipper: |
| 21 | """ |
| 22 | Creates gradient clipping closure to clip by value or by norm, |
| 23 | according to the provided config. |
| 24 | """ |
| 25 | cfg = cfg.clone() |
| 26 | |
| 27 | def clip_grad_norm(p: _GradientClipperInput): |
| 28 | torch.nn.utils.clip_grad_norm_(p, cfg.CLIP_VALUE, cfg.NORM_TYPE) |
| 29 | |
| 30 | def clip_grad_value(p: _GradientClipperInput): |
| 31 | torch.nn.utils.clip_grad_value_(p, cfg.CLIP_VALUE) |
| 32 | |
| 33 | _GRADIENT_CLIP_TYPE_TO_CLIPPER = { |
| 34 | GradientClipType.VALUE: clip_grad_value, |
| 35 | GradientClipType.NORM: clip_grad_norm, |
| 36 | } |
| 37 | return _GRADIENT_CLIP_TYPE_TO_CLIPPER[GradientClipType(cfg.CLIP_TYPE)] |
| 38 | |
| 39 | |
| 40 | def _generate_optimizer_class_with_gradient_clipping( |
no test coverage detected