Method _rms

PATH/core/optimizers/adafactor.py:114–115

(self, tensor: torch.Tensor)

        return factored, use_first_moment

    def _rms(self, tensor: torch.Tensor) -> float:
        return tensor.norm(2) / (tensor.numel() ** 0.5)

    def _approx_sq_grad(
        self,