| 292 | |
| 293 | |
| 294 | class SgdOptimizer(Optimizer): |
| 295 | def __init__( |
| 296 | self, |
| 297 | base_learning_rate=0.01, |
| 298 | policy="fixed", |
| 299 | momentum=0.0, |
| 300 | nesterov=True, |
| 301 | sparse_dedup_aggregator=None, |
| 302 | lars=None, |
| 303 | **kwargs |
| 304 | ): |
| 305 | super().__init__() |
| 306 | self.base_learning_rate = base_learning_rate |
| 307 | self.policy = policy |
| 308 | self.momentum = momentum |
| 309 | self.nesterov = nesterov |
| 310 | self.sparse_dedup_aggregator = sparse_dedup_aggregator |
| 311 | self.lars = lars |
| 312 | self.init_kwargs = kwargs |
| 313 | |
| 314 | def _run(self, net, param_init_net, param_info): |
| 315 | param = param_info.blob |
| 316 | grad = param_info.grad |
| 317 | if self.base_learning_rate == 0: |
| 318 | return |
| 319 | assert ( |
| 320 | self.base_learning_rate > 0 |
| 321 | ), "Expect positive base learning rate, got {}".format(self.base_learning_rate) |
| 322 | |
| 323 | self._clear_local_lr_multiplier() |
| 324 | |
| 325 | # TODO(zqq): support LARS for sparse parameters |
| 326 | if self.lars is not None and not isinstance(grad, core.GradientSlice): |
| 327 | assert self.lars >= 0, "Lars offset must be nonnegative, got {}".format( |
| 328 | self.lars |
| 329 | ) |
| 330 | wd, trust, lr_max = self.create_lars_inputs( |
| 331 | param_init_net, 0.0, 1.0, np.finfo(np.float32).max |
| 332 | ) |
| 333 | lr_lars_multiplier = net.Lars( |
| 334 | [param, grad, wd, trust, lr_max], |
| 335 | self.make_unique_blob_name(str(param) + "_lars"), |
| 336 | offset=self.lars, |
| 337 | lr_min=0.0, |
| 338 | ) |
| 339 | current_scope = scope.CurrentDeviceScope() |
| 340 | self._add_local_lr_multiplier( |
| 341 | lr_lars_multiplier, |
| 342 | is_gpu_blob=( |
| 343 | current_scope is not None |
| 344 | and core.IsGPUDeviceType(current_scope.device_type) |
| 345 | ), |
| 346 | ) |
| 347 | |
| 348 | # We need negative sign for LR when used directly with WeightedSum |
| 349 | # below. |
| 350 | lr_sign = -1 if self.momentum else 1 |
| 351 | lr, _ = self.build_lr( |
no outgoing calls
searching dependent graphs…