| 451 | ''' |
| 452 | |
| 453 | def __init__(self, lr=0.1, epsilon=1e-8, weight_decay=0): |
| 454 | super(AdaGrad, self).__init__(lr) |
| 455 | |
| 456 | # init weight_decay |
| 457 | if type(weight_decay) == float or type(weight_decay) == int: |
| 458 | if weight_decay < 0.0: |
| 459 | raise ValueError( |
| 460 | "Invalid weight_decay value: {}".format(weight_decay)) |
| 461 | self.weight_decay = Constant(weight_decay) |
| 462 | elif isinstance(weight_decay, DecayScheduler): |
| 463 | self.weight_decay = weight_decay |
| 464 | else: |
| 465 | raise TypeError("Wrong weight_decay type") |
| 466 | self.decay_value = self.weight_decay(self.step_counter) |
| 467 | |
| 468 | # init epsilon |
| 469 | if type(epsilon) == float or type(epsilon) == int: |
| 470 | self.epsilon = Constant(epsilon) |
| 471 | elif isinstance(epsilon, DecayScheduler): |
| 472 | self.epsilon = epsilon |
| 473 | else: |
| 474 | raise TypeError("Wrong epsilon type") |
| 475 | self.epsilon_value = self.epsilon(self.step_counter) |
| 476 | |
| 477 | # init history |
| 478 | self.history = dict() |
| 479 | |
| 480 | def apply(self, param_name, param_value, param_grad): |
| 481 | """Performs a single optimization step. |