r"""Unscale all ``grad_tensors``'s grad. Args: grad_tensors: Tensors needed to unscale grads. Should be all tensors that are affected by ``target`` tensor in GradManager's backward.
(self, grad_tensors: Iterable[Tensor])
| 118 | self.update() |
| 119 | |
| 120 | def unscale(self, grad_tensors: Iterable[Tensor]): |
| 121 | r"""Unscale all ``grad_tensors``'s grad. |
| 122 | |
| 123 | Args: |
| 124 | grad_tensors: Tensors needed to unscale grads. Should be all tensors |
| 125 | that are affected by ``target`` tensor in GradManager's backward. |
| 126 | """ |
| 127 | if self.growth_interval == 0: |
| 128 | # use float64 for better precision |
| 129 | inv_scale = Tensor(1.0 / self.scale_factor) |
| 130 | for tensor in grad_tensors: |
| 131 | if tensor is None or getattr(tensor, "grad", None) is None: |
| 132 | continue |
| 133 | tensor.grad *= inv_scale |
| 134 | return self |
| 135 | |
| 136 | # to support tracing, _check_gradients should be applied to every grad. |
| 137 | if self._check_gradients( |
| 138 | [x.grad for x in grad_tensors if x.grad is not None], |
| 139 | 1.0 / self.scale_factor, |
| 140 | ).numpy(): |
| 141 | self._found_non_finite = True |
| 142 | for tensor in grad_tensors: |
| 143 | if tensor is None or getattr(tensor, "grad", None) is None: |
| 144 | continue |
| 145 | tensor.grad = None |
| 146 | return self |
| 147 | |
| 148 | def _check_gradients(self, grads, scale): |
| 149 | if len(grads) == 0: |