(self, net, param_init_net, param_info)
| 748 | self.use_mask = True |
| 749 | |
| 750 | def _run(self, net, param_init_net, param_info): |
| 751 | param = param_info.blob |
| 752 | grad = param_info.grad |
| 753 | |
| 754 | if self.alpha <= 0: |
| 755 | return |
| 756 | |
| 757 | self._clear_local_lr_multiplier() |
| 758 | |
| 759 | if self.lars is not None and not isinstance(grad, core.GradientSlice): |
| 760 | assert ( |
| 761 | self.weight_decay == 0 |
| 762 | ), "weight decay is not implemented for LARS yet" |
| 763 | assert self.lars >= 0, "Lars offset must be nonnegative, got {}".format( |
| 764 | self.lars |
| 765 | ) |
| 766 | wd, trust, lr_max = self.create_lars_inputs( |
| 767 | param_init_net, 0.0, 1.0, np.finfo(np.float32).max |
| 768 | ) |
| 769 | lr_lars_multiplier = net.Lars( |
| 770 | [param, grad, wd, trust, lr_max], |
| 771 | self.make_unique_blob_name(str(param) + "_lars"), |
| 772 | offset=self.lars, |
| 773 | lr_min=0.0, |
| 774 | ) |
| 775 | |
| 776 | current_scope = scope.CurrentDeviceScope() |
| 777 | self._add_local_lr_multiplier( |
| 778 | lr_lars_multiplier, |
| 779 | is_gpu_blob=( |
| 780 | current_scope is not None |
| 781 | and core.IsGPUDeviceType(current_scope.device_type) |
| 782 | ), |
| 783 | ) |
| 784 | |
| 785 | lr, lr_iteration = self.build_lr( |
| 786 | net, |
| 787 | param_init_net, |
| 788 | base_learning_rate=self.alpha, |
| 789 | policy=self.policy, |
| 790 | **(self.init_kwargs) |
| 791 | ) |
| 792 | iteration = ( |
| 793 | self.build_non_lr_iter(net, param_init_net, iter_val=0) |
| 794 | if self._use_dedicated_lr_iteration_counter |
| 795 | else lr_iteration |
| 796 | ) |
| 797 | |
| 798 | if self.counter_halflife > 0: |
| 799 | self._aux_params.shared.append(iteration) |
| 800 | |
| 801 | if self.rowWise: |
| 802 | logger.debug( |
| 803 | "Using engine {} for rowWise Adagrad to train param {}".format( |
| 804 | self.engine, param |
| 805 | ) |
| 806 | ) |
| 807 |
nothing calls this directly
no test coverage detected