| 344 | ''' |
| 345 | |
| 346 | def __init__(self, lr=0.1, rho=0.9, epsilon=1e-8, weight_decay=0): |
| 347 | super(RMSProp, self).__init__(lr) |
| 348 | |
| 349 | # init weight_decay |
| 350 | if type(weight_decay) == float or type(weight_decay) == int: |
| 351 | if weight_decay < 0.0: |
| 352 | raise ValueError( |
| 353 | "Invalid weight_decay value: {}".format(weight_decay)) |
| 354 | self.weight_decay = Constant(weight_decay) |
| 355 | elif isinstance(weight_decay, DecayScheduler): |
| 356 | self.weight_decay = weight_decay |
| 357 | else: |
| 358 | raise TypeError("Wrong weight_decay type") |
| 359 | self.decay_value = self.weight_decay(self.step_counter) |
| 360 | |
| 361 | # init rho |
| 362 | if type(rho) == float or type(rho) == int: |
| 363 | self.rho = Constant(rho) |
| 364 | elif isinstance(rho, DecayScheduler): |
| 365 | self.rho = rho |
| 366 | else: |
| 367 | raise TypeError("Wrong rho type") |
| 368 | self.rho_value = self.rho(self.step_counter) |
| 369 | |
| 370 | # init epsilon |
| 371 | if type(epsilon) == float or type(epsilon) == int: |
| 372 | self.epsilon = Constant(epsilon) |
| 373 | elif isinstance(rho, DecayScheduler): |
| 374 | self.epsilon = epsilon |
| 375 | else: |
| 376 | raise TypeError("Wrong epsilon type") |
| 377 | self.epsilon_value = self.epsilon(self.step_counter) |
| 378 | |
| 379 | # init running average |
| 380 | self.running_average = dict() |
| 381 | |
| 382 | def apply(self, param_name, param_value, param_grad): |
| 383 | """Performs a single optimization step. |