| 21 | # helper for adam optimizer |
| 22 | # use tensorflow defaults |
| 23 | def adam(cost, params, lr0=1e-3, beta1=0.9, beta2=0.999, eps=1e-8): |
| 24 | grads = T.grad(cost, params) |
| 25 | updates = [] |
| 26 | time = theano.shared(0) |
| 27 | new_time = time + 1 |
| 28 | updates.append((time, new_time)) |
| 29 | lr = lr0*T.sqrt(1 - beta2**new_time) / (1 - beta1**new_time) |
| 30 | for p, g in zip(params, grads): |
| 31 | m = theano.shared(p.get_value() * 0.) |
| 32 | v = theano.shared(p.get_value() * 0.) |
| 33 | new_m = beta1*m + (1 - beta1)*g |
| 34 | new_v = beta2*v + (1 - beta2)*g*g |
| 35 | new_p = p - lr*new_m / (T.sqrt(new_v) + eps) |
| 36 | updates.append((m, new_m)) |
| 37 | updates.append((v, new_v)) |
| 38 | updates.append((p, new_p)) |
| 39 | return updates |
| 40 | |
| 41 | |
| 42 | # so you can test different architectures |