Class AdadeltaOptimizer

caffe2/python/optimizer.py:1399–1478

class AdadeltaOptimizer(Optimizer):
    def __init__(
        self,
        alpha=0.01,
        epsilon=1e-4,
        decay=0.95,
        policy="fixed",
        sparse_dedup_aggregator=None,
        engine="",
        **kwargs
    ):
        """Constructor function to add Adadelta Optimizer

        Args:
            alpha: learning rate
            epsilon: attribute of Adadelta to avoid numerical issues
            decay: attribute of Adadelta to decay the squared gradient sum
            policy: specifies how learning rate should be applied, options are
              "fixed", "step", "exp", etc.
            sparse_dedup_aggregator: specifies deduplication strategy for
              gradient slices. Works while using sparse gradients. Options
              include "mean" and "sum".
            engine: the engine used, options include "", "CUDNN", etc.
        """
        super().__init__()
        self.alpha = alpha
        self.epsilon = epsilon
        self.decay = decay
        self.policy = policy
        self.sparse_dedup_aggregator = sparse_dedup_aggregator
        self.engine = engine
        self.init_kwargs = kwargs

    def _run(self, net, param_init_net, param_info):
        param = param_info.blob
        grad = param_info.grad

        if self.alpha <= 0:
            return

        lr, _ = self.build_lr(
            net,
            param_init_net,
            base_learning_rate=self.alpha,
            policy=self.policy,
            **(self.init_kwargs)
        )

        moment = param_init_net.ConstantFill(
            [param], str(param) + "_squared_moment", value=0.0
        )

        moment_update = param_init_net.ConstantFill(
            [param], str(param) + "_squared_moment_update", value=0.0
        )

        self._aux_params.local.append(moment)
        self._aux_params.local.append(moment_update)