(self, net, param_init_net, param_info)
| 1330 | self.init_kwargs = kwargs |
| 1331 | |
| 1332 | def _run(self, net, param_init_net, param_info): |
| 1333 | param = param_info.blob |
| 1334 | grad = param_info.grad |
| 1335 | |
| 1336 | if self.lr <= 0: |
| 1337 | return |
| 1338 | |
| 1339 | self._clear_local_lr_multiplier() |
| 1340 | |
| 1341 | if self.lars is not None and not isinstance(grad, core.GradientSlice): |
| 1342 | assert self.lars >= 0, "Lars offset must be nonnegative, got {}".format( |
| 1343 | self.lars |
| 1344 | ) |
| 1345 | wd, trust, lr_max = self.create_lars_inputs( |
| 1346 | param_init_net, 0.0, 1.0, np.finfo(np.float32).max |
| 1347 | ) |
| 1348 | lr_lars_multiplier = net.Lars( |
| 1349 | [param, grad, wd, trust, lr_max], |
| 1350 | self.make_unique_blob_name(str(param) + "_lars"), |
| 1351 | offset=self.lars, |
| 1352 | lr_min=0.0, |
| 1353 | ) |
| 1354 | current_scope = scope.CurrentDeviceScope() |
| 1355 | self._add_local_lr_multiplier( |
| 1356 | lr_lars_multiplier, |
| 1357 | is_gpu_blob=( |
| 1358 | current_scope is not None |
| 1359 | and core.IsGPUDeviceType(current_scope.device_type) |
| 1360 | ), |
| 1361 | ) |
| 1362 | |
| 1363 | lr, _ = self.build_lr( |
| 1364 | net, |
| 1365 | param_init_net, |
| 1366 | base_learning_rate=self.lr, |
| 1367 | policy=self.policy, |
| 1368 | **(self.init_kwargs) |
| 1369 | ) |
| 1370 | |
| 1371 | moment = param_init_net.ConstantFill(param, str(param) + "_moment", value=0.0) |
| 1372 | self._aux_params.local.append(moment) |
| 1373 | |
| 1374 | grad_sq_sum = param_init_net.ConstantFill( |
| 1375 | [], str(param) + "_grad_sq_sum", shape=[1], value=self.grad_sq_init |
| 1376 | ) |
| 1377 | self._aux_params.local.append(grad_sq_sum) |
| 1378 | |
| 1379 | if isinstance(grad, core.GradientSlice): |
| 1380 | grad = self.dedup(net, self.sparse_dedup_aggregator, grad) |
| 1381 | net.SparseStorm( |
| 1382 | [param, moment, grad_sq_sum, grad.values, grad.indices, lr], |
| 1383 | [param, moment, grad_sq_sum], |
| 1384 | momentum=self.momentum, |
| 1385 | beta=self.beta, |
| 1386 | ) |
| 1387 | else: |
| 1388 | net.Storm( |
| 1389 | [param, moment, grad_sq_sum, grad, lr], |
nothing calls this directly
no test coverage detected