| 1194 | |
| 1195 | |
| 1196 | class WngradOptimizer(Optimizer): |
| 1197 | def __init__( |
| 1198 | self, |
| 1199 | alpha=1.0, |
| 1200 | epsilon=1e-9, |
| 1201 | policy="fixed", |
| 1202 | sparse_dedup_aggregator=None, |
| 1203 | engine="", |
| 1204 | moment_init=100.0, |
| 1205 | lars=None, |
| 1206 | output_effective_lr=False, |
| 1207 | output_effective_lr_and_update=False, |
| 1208 | **kwargs |
| 1209 | ): |
| 1210 | super().__init__() |
| 1211 | self.alpha = alpha |
| 1212 | self.epsilon = epsilon |
| 1213 | self.policy = policy |
| 1214 | self.sparse_dedup_aggregator = sparse_dedup_aggregator |
| 1215 | self.engine = engine |
| 1216 | self.moment_init = moment_init |
| 1217 | self.lars = lars |
| 1218 | self.output_effective_lr = output_effective_lr |
| 1219 | self.output_effective_lr_and_update = output_effective_lr_and_update |
| 1220 | self.init_kwargs = kwargs |
| 1221 | |
| 1222 | def _run(self, net, param_init_net, param_info): |
| 1223 | param = param_info.blob |
| 1224 | grad = param_info.grad |
| 1225 | |
| 1226 | if self.alpha <= 0: |
| 1227 | return |
| 1228 | |
| 1229 | self._clear_local_lr_multiplier() |
| 1230 | |
| 1231 | if self.lars is not None and not isinstance(grad, core.GradientSlice): |
| 1232 | assert self.lars >= 0, "Lars offset must be nonnegative, got {}".format( |
| 1233 | self.lars |
| 1234 | ) |
| 1235 | wd, trust, lr_max = self.create_lars_inputs( |
| 1236 | param_init_net, 0.0, 1.0, np.finfo(np.float32).max |
| 1237 | ) |
| 1238 | lr_lars_multiplier = net.Lars( |
| 1239 | [param, grad, wd, trust, lr_max], |
| 1240 | self.make_unique_blob_name(str(param) + "_lars"), |
| 1241 | offset=self.lars, |
| 1242 | lr_min=0.0, |
| 1243 | ) |
| 1244 | current_scope = scope.CurrentDeviceScope() |
| 1245 | self._add_local_lr_multiplier( |
| 1246 | lr_lars_multiplier, |
| 1247 | is_gpu_blob=( |
| 1248 | current_scope is not None |
| 1249 | and core.IsGPUDeviceType(current_scope.device_type) |
| 1250 | ), |
| 1251 | ) |
| 1252 | |
| 1253 | lr, _ = self.build_lr( |
no outgoing calls
no test coverage detected
searching dependent graphs…