(self, net, param_init_net, param_info)
| 312 | self.init_kwargs = kwargs |
| 313 | |
| 314 | def _run(self, net, param_init_net, param_info): |
| 315 | param = param_info.blob |
| 316 | grad = param_info.grad |
| 317 | if self.base_learning_rate == 0: |
| 318 | return |
| 319 | assert ( |
| 320 | self.base_learning_rate > 0 |
| 321 | ), "Expect positive base learning rate, got {}".format(self.base_learning_rate) |
| 322 | |
| 323 | self._clear_local_lr_multiplier() |
| 324 | |
| 325 | # TODO(zqq): support LARS for sparse parameters |
| 326 | if self.lars is not None and not isinstance(grad, core.GradientSlice): |
| 327 | assert self.lars >= 0, "Lars offset must be nonnegative, got {}".format( |
| 328 | self.lars |
| 329 | ) |
| 330 | wd, trust, lr_max = self.create_lars_inputs( |
| 331 | param_init_net, 0.0, 1.0, np.finfo(np.float32).max |
| 332 | ) |
| 333 | lr_lars_multiplier = net.Lars( |
| 334 | [param, grad, wd, trust, lr_max], |
| 335 | self.make_unique_blob_name(str(param) + "_lars"), |
| 336 | offset=self.lars, |
| 337 | lr_min=0.0, |
| 338 | ) |
| 339 | current_scope = scope.CurrentDeviceScope() |
| 340 | self._add_local_lr_multiplier( |
| 341 | lr_lars_multiplier, |
| 342 | is_gpu_blob=( |
| 343 | current_scope is not None |
| 344 | and core.IsGPUDeviceType(current_scope.device_type) |
| 345 | ), |
| 346 | ) |
| 347 | |
| 348 | # We need negative sign for LR when used directly with WeightedSum |
| 349 | # below. |
| 350 | lr_sign = -1 if self.momentum else 1 |
| 351 | lr, _ = self.build_lr( |
| 352 | net, |
| 353 | param_init_net, |
| 354 | base_learning_rate=self.base_learning_rate * lr_sign, |
| 355 | policy=self.policy, |
| 356 | **(self.init_kwargs) |
| 357 | ) |
| 358 | |
| 359 | dev = scope.CurrentDeviceScope() |
| 360 | if dev is None: |
| 361 | dev = core.DeviceOption(caffe2_pb2.CPU) |
| 362 | |
| 363 | # Each GPU/CPU must have its own ONE blob, thus modify the name |
| 364 | # to include device information. |
| 365 | ONE = param_init_net.ConstantFill( |
| 366 | [], |
| 367 | "ONE_{}_{}{}".format(dev.device_type, dev.device_id, dev.node_name), |
| 368 | shape=[1], |
| 369 | value=1.0, |
| 370 | ) |
| 371 |
nothing calls this directly
no test coverage detected