(
self,
net,
param_init_net,
base_learning_rate,
learning_rate_blob=None,
policy="fixed",
iter_val=0,
**kwargs
)
| 113 | return self.get_cpu_blob_name(base_str, current_scope.node_name) |
| 114 | |
| 115 | def build_lr( |
| 116 | self, |
| 117 | net, |
| 118 | param_init_net, |
| 119 | base_learning_rate, |
| 120 | learning_rate_blob=None, |
| 121 | policy="fixed", |
| 122 | iter_val=0, |
| 123 | **kwargs |
| 124 | ): |
| 125 | if learning_rate_blob is None: |
| 126 | learning_rate_blob = self.make_unique_blob_name("lr") |
| 127 | |
| 128 | if self._use_dedicated_lr_iteration_counter: |
| 129 | iteration = utils.BuildUniqueMutexIter( |
| 130 | param_init_net, |
| 131 | net, |
| 132 | iter=utils.OPTIMIZER_ITERATION_LR_NAME, |
| 133 | iter_mutex=utils.ITERATION_MUTEX_LR_NAME, |
| 134 | iter_val=iter_val, |
| 135 | ) |
| 136 | logger.info(f"Created dedicated learning rate iteration counter: {iteration}") |
| 137 | else: |
| 138 | iteration = utils.BuildUniqueMutexIter(param_init_net, net, iter_val=iter_val) |
| 139 | |
| 140 | if not net.BlobIsDefined(learning_rate_blob): |
| 141 | # There is one interesting thing here: since we are minimizing, we are |
| 142 | # doing "descent" so the learning rate is set to be negative. |
| 143 | lr = net.LearningRate( |
| 144 | [iteration], |
| 145 | learning_rate_blob, |
| 146 | base_lr=-base_learning_rate, |
| 147 | policy=policy, |
| 148 | **kwargs |
| 149 | ) |
| 150 | else: |
| 151 | lr = net.GetBlobRef(learning_rate_blob) |
| 152 | |
| 153 | if self._lr_multiplier is not None: |
| 154 | lr_multiplier = net.CopyFromCPUInput( |
| 155 | self._lr_multiplier, self.make_unique_blob_name("lr_multiplier") |
| 156 | ) |
| 157 | |
| 158 | lr = net.Mul( |
| 159 | [lr, lr_multiplier], |
| 160 | self.make_unique_blob_name("scaled_lr"), |
| 161 | broadcast=1, |
| 162 | ) |
| 163 | |
| 164 | if self._local_lr_multiplier is not None: |
| 165 | current_scope = scope.CurrentDeviceScope() |
| 166 | if ( |
| 167 | current_scope is not None |
| 168 | and core.IsGPUDeviceType(current_scope.device_type) |
| 169 | and not self._local_lr_multiplier_on_gpu |
| 170 | ): |
| 171 | local_lr_multiplier = net.CopyFromCPUInput( |
| 172 | self._local_lr_multiplier, |
no test coverage detected