(self, net, param_init_net, param_info)
| 1220 | self.init_kwargs = kwargs |
| 1221 | |
| 1222 | def _run(self, net, param_init_net, param_info): |
| 1223 | param = param_info.blob |
| 1224 | grad = param_info.grad |
| 1225 | |
| 1226 | if self.alpha <= 0: |
| 1227 | return |
| 1228 | |
| 1229 | self._clear_local_lr_multiplier() |
| 1230 | |
| 1231 | if self.lars is not None and not isinstance(grad, core.GradientSlice): |
| 1232 | assert self.lars >= 0, "Lars offset must be nonnegative, got {}".format( |
| 1233 | self.lars |
| 1234 | ) |
| 1235 | wd, trust, lr_max = self.create_lars_inputs( |
| 1236 | param_init_net, 0.0, 1.0, np.finfo(np.float32).max |
| 1237 | ) |
| 1238 | lr_lars_multiplier = net.Lars( |
| 1239 | [param, grad, wd, trust, lr_max], |
| 1240 | self.make_unique_blob_name(str(param) + "_lars"), |
| 1241 | offset=self.lars, |
| 1242 | lr_min=0.0, |
| 1243 | ) |
| 1244 | current_scope = scope.CurrentDeviceScope() |
| 1245 | self._add_local_lr_multiplier( |
| 1246 | lr_lars_multiplier, |
| 1247 | is_gpu_blob=( |
| 1248 | current_scope is not None |
| 1249 | and core.IsGPUDeviceType(current_scope.device_type) |
| 1250 | ), |
| 1251 | ) |
| 1252 | |
| 1253 | lr, _ = self.build_lr( |
| 1254 | net, |
| 1255 | param_init_net, |
| 1256 | base_learning_rate=self.alpha, |
| 1257 | policy=self.policy, |
| 1258 | **(self.init_kwargs) |
| 1259 | ) |
| 1260 | |
| 1261 | moment = param_init_net.ConstantFill( |
| 1262 | [], str(param) + "_moment", shape=[1], value=self.moment_init |
| 1263 | ) |
| 1264 | |
| 1265 | self._aux_params.local.append(moment) |
| 1266 | |
| 1267 | if isinstance(grad, core.GradientSlice): |
| 1268 | grad = self.dedup(net, self.sparse_dedup_aggregator, grad) |
| 1269 | net.SparseWngrad( |
| 1270 | [param, moment, grad.indices, grad.values, lr], |
| 1271 | [param, moment], |
| 1272 | epsilon=self.epsilon, |
| 1273 | engine=self.engine, |
| 1274 | ) |
| 1275 | else: |
| 1276 | output_args = [param, moment] |
| 1277 | if self.output_effective_lr_and_update: |
| 1278 | output_args.append(str(param) + "_effective_lr") |
| 1279 | output_args.append(str(param) + "_update") |
nothing calls this directly
no test coverage detected