(cost, params, lr, eps=1e-10)
| 19 | |
| 20 | |
| 21 | def adagrad(cost, params, lr, eps=1e-10): |
| 22 | grads = T.grad(cost, params) |
| 23 | caches = [theano.shared(np.ones_like(p.get_value())) for p in params] |
| 24 | new_caches = [c + g*g for c, g in zip(caches, grads)] |
| 25 | |
| 26 | c_update = [(c, new_c) for c, new_c in zip(caches, new_caches)] |
| 27 | g_update = [ |
| 28 | (p, p - lr*g / T.sqrt(new_c + eps)) for p, new_c, g in zip(params, new_caches, grads) |
| 29 | ] |
| 30 | updates = c_update + g_update |
| 31 | return updates |
| 32 | |
| 33 | |
| 34 | class RecursiveNN: |