Method clip_grad

python/oneflow/optim/optimizer.py:406–434

r"""Clips gradient norm of an iterable of parameters. The norm is computed over all gradients together, as if they were concatenated into a single vector. You can set the max_norm and norm_type. For more details, you can refer to the documentation of each optimizer(like A

(self, error_if_nonfinite: bool = False)

        raise NotImplementedError()

    def clip_grad(self, error_if_nonfinite: bool = False):
        r"""Clips gradient norm of an iterable of parameters. 
        The norm is computed over all gradients together, as if they were concatenated into a single vector.

        You can set the max_norm and norm_type. 

        For more details, you can refer to the documentation of each optimizer(like Adam, SGD and so on). 

        You can also refer the code in :func:`oneflow.nn.utils.clip_grad_norm_`

        Args:
            error_if_nonfinite (bool): if True, an error is thrown if the total
                norm of the gradients from :attr:``parameters`` is ``nan``,
                ``inf``, or ``-inf``. Default: False (will switch to True in the future)

        """
        for param_group in self.param_groups:
            if param_group._enable_clip_grad:
                clip_grad_norm_(
                    param_group.parameters,
                    param_group["clip_grad_max_norm"],
                    param_group["clip_grad_norm_type"],
                    error_if_nonfinite,
                    param_group.get("fused", False),
                )
            else:
                warnings.warn(
                    "To enable clip_grad, passing the `clip_grad_max_norm` and `clip_grad_norm_type` parameters when instantializing the Optimizer."
                )

    def zero_grad(self, set_to_none: bool = False):
        """Sets the gradients of all optimized :class:`oneflow.Tensor` s to zero.