| 77 | |
| 78 | # theano |
| 79 | def rmsprop(cost, params, lr=1e-3, decay=0.9, eps=1e-8): |
| 80 | # return updates |
| 81 | lr = np.float32(lr) |
| 82 | decay = np.float32(decay) |
| 83 | eps = np.float32(eps) |
| 84 | |
| 85 | updates = [] |
| 86 | grads = T.grad(cost, params) |
| 87 | |
| 88 | # tf-like |
| 89 | # caches = [theano.shared(np.ones_like(p.get_value(), dtype=np.float32)) for p in params] |
| 90 | |
| 91 | # keras-like |
| 92 | caches = [theano.shared(np.zeros_like(p.get_value(), dtype=np.float32)) for p in params] |
| 93 | |
| 94 | new_caches = [] |
| 95 | for c, g in zip(caches, grads): |
| 96 | new_c = decay*c + (np.float32(1) - decay)*g*g |
| 97 | updates.append((c, new_c)) |
| 98 | new_caches.append(new_c) |
| 99 | |
| 100 | for p, new_c, g in zip(params, new_caches, grads): |
| 101 | new_p = p - lr*g / T.sqrt(new_c + eps) |
| 102 | updates.append((p, new_p)) |
| 103 | |
| 104 | return updates |
| 105 | |
| 106 | thX = T.matrix('X') |
| 107 | thY = T.matrix('Y') |