(cost, params, lr, eps=1e-10)
| 69 | |
| 70 | |
| 71 | def adagrad(cost, params, lr, eps=1e-10): |
| 72 | grads = T.grad(cost, params) |
| 73 | caches = [theano.shared(np.ones_like(p.get_value())) for p in params] |
| 74 | new_caches = [c + g*g for c, g in zip(caches, grads)] |
| 75 | |
| 76 | c_update = [(c, new_c) for c, new_c in zip(caches, new_caches)] |
| 77 | g_update = [ |
| 78 | (p, p - lr*g / T.sqrt(new_c + eps)) for p, new_c, g in zip(params, new_caches, grads) |
| 79 | ] |
| 80 | updates = c_update + g_update |
| 81 | return updates |
| 82 | |
| 83 | |
| 84 | class RecursiveNN: |