(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-6, weight_decay=0.0,
clip_norm=1.0, param_init_type=mstype.float32)
| 134 | """ |
| 135 | |
| 136 | def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-6, weight_decay=0.0, |
| 137 | clip_norm=1.0, param_init_type=mstype.float32): |
| 138 | super(AdamWeightDecayOp, self).__init__(learning_rate, params, weight_decay) |
| 139 | _check_param_value(beta1, beta2, eps, self.cls_name) |
| 140 | self.beta1 = Tensor(np.array([beta1]).astype(np.float32)) |
| 141 | self.beta2 = Tensor(np.array([beta2]).astype(np.float32)) |
| 142 | self.eps = Tensor(np.array([eps]).astype(np.float32)) |
| 143 | self.clip_norm = Tensor([clip_norm], mstype.float32) |
| 144 | self.enable_init_fp16 = (param_init_type == mstype.float16) |
| 145 | if self.enable_init_fp16: |
| 146 | self.moments1 = self.clone_param32(prefix="adam_m", init='zeros') |
| 147 | self.moments2 = self.clone_param32(prefix="adam_v", init='zeros') |
| 148 | self.opt = P.FusedCastAdamWeightDecay() |
| 149 | else: |
| 150 | self.moments1 = self.parameters.clone(prefix="adam_m", init='zeros') |
| 151 | self.moments2 = self.parameters.clone(prefix="adam_v", init='zeros') |
| 152 | self.opt = P.AdamWeightDecay() |
| 153 | self.hyper_map = C.HyperMap() |
| 154 | self.opt.add_prim_attr("primitive_target", "CPU") |
| 155 | |
| 156 | def construct(self, gradients, clip_value): |
| 157 | """AdamWeightDecayOp""" |
nothing calls this directly
no test coverage detected