Class WeightDecayBuilder

caffe2/python/optimizer.py:589–613

class WeightDecayBuilder(Optimizer):
    def __init__(self, weight_decay):
        self.weight_decay = weight_decay

    def _run(self, net, param_init_net, param_info):
        dev = scope.CurrentDeviceScope()
        if dev is None:
            dev = core.DeviceOption(caffe2_pb2.CPU)

        ONE = param_init_net.ConstantFill(
            [], "ONE_{}_{}".format(dev.device_type, dev.device_id), shape=[1], value=1.0
        )
        WD = param_init_net.ConstantFill(
            [],
            "wd_{}_{}".format(dev.device_type, dev.device_id),
            shape=[1],
            value=self.weight_decay,
        )

        if isinstance(param_info.grad, core.GradientSlice):
            raise ValueError("Weight decay does not yet support sparse gradients")
        else:
            net.WeightedSum(
                [param_info.grad, ONE, param_info.blob, WD], param_info.grad
            )


class AdagradOptimizer(Optimizer):