Method _run

caffe2/python/optimizer.py:1793–1864

(self, net, param_init_net, param_info)

            )

    def _run(self, net, param_init_net, param_info):
        param = param_info.blob
        grad = param_info.grad

        if self.alpha <= 0:
            return

        lr, iteration = self.build_lr(
            net,
            param_init_net,
            base_learning_rate=self.alpha,
            policy=self.policy,
            **(self.init_kwargs)
        )

        if isinstance(grad, core.GradientSlice):
            # hack for position weighted.
            param_squared_sum = param_init_net.ConstantFill([param], param + "_squared_sum", value=0.0)
            self._aux_params.local.append(param_squared_sum)
            output_blobs = [param, param_squared_sum]
            net.SparseAdagrad(
                [param, param_squared_sum, grad.indices, grad.values, lr],
                output_blobs,
                epsilon=self.epsilon,
            )
        else:
            m1 = param_init_net.ConstantFill([param], param + "_first_mo1ment", value=0.0)
            m2 = param_init_net.ConstantFill([param], param + "_second_moment", value=0.0)
            self._aux_params.shared.append(iteration)
            self._aux_params.local.append(m1)
            self._aux_params.local.append(m2)
            output_blobs = [param, m1, m2]
            net.DecayAdagrad(
                [param, m1, m2, grad, lr, iteration],
                output_blobs,
                beta1=self.beta1,
                beta2=self.beta2,
                epsilon=self.epsilon,
                weight_decay=self.weight_decay,
                bias_correction_first=self.bias_correction_first,
            )

            if self.ema_enabled:
                param_ema = str(param) + "_ema"
                if not param_init_net.BlobIsDefined(param_ema):
                    param_init_net.ConstantFill([param], param_ema, value=0.0)
                    self._aux_params.local.append(param_ema)

                net.EMA(
                    [param, param_ema, iteration],
                    [param, param_ema],
                    ema_start=self.ema_start,
                    ema_end=self.ema_end,
                    ema_step=self.ema_step,
                    ema_alpha=self.ema_alpha,
                )

            if self.ema_teacher_enabled: