(self, loss, params)
| 564 | self.initial_decay = decay |
| 565 | |
| 566 | def get_updates(self, loss, params): |
| 567 | grads = self.get_gradients(loss, params) |
| 568 | self.updates = [] |
| 569 | |
| 570 | lr = self.lr |
| 571 | if self.initial_decay > 0: |
| 572 | lr = lr * ( # pylint: disable=g-no-augmented-assignment |
| 573 | 1. / |
| 574 | (1. + |
| 575 | self.decay * math_ops.cast(self.iterations, K.dtype(self.decay)))) |
| 576 | |
| 577 | with ops.control_dependencies([state_ops.assign_add(self.iterations, 1)]): |
| 578 | t = math_ops.cast(self.iterations, K.floatx()) |
| 579 | lr_t = lr / (1. - math_ops.pow(self.beta_1, t)) |
| 580 | |
| 581 | shapes = [K.int_shape(p) for p in params] |
| 582 | # zero init of 1st moment |
| 583 | ms = [K.zeros(shape) for shape in shapes] |
| 584 | # zero init of exponentially weighted infinity norm |
| 585 | us = [K.zeros(shape) for shape in shapes] |
| 586 | self.weights = [self.iterations] + ms + us |
| 587 | |
| 588 | for p, g, m, u in zip(params, grads, ms, us): |
| 589 | |
| 590 | m_t = (self.beta_1 * m) + (1. - self.beta_1) * g |
| 591 | u_t = math_ops.maximum(self.beta_2 * u, math_ops.abs(g)) |
| 592 | p_t = p - lr_t * m_t / (u_t + self.epsilon) |
| 593 | |
| 594 | self.updates.append(state_ops.assign(m, m_t)) |
| 595 | self.updates.append(state_ops.assign(u, u_t)) |
| 596 | new_p = p_t |
| 597 | |
| 598 | # Apply constraints. |
| 599 | if getattr(p, 'constraint', None) is not None: |
| 600 | new_p = p.constraint(new_p) |
| 601 | |
| 602 | self.updates.append(state_ops.assign(p, new_p)) |
| 603 | return self.updates |
| 604 | |
| 605 | def get_config(self): |
| 606 | config = { |
nothing calls this directly
no test coverage detected