| 28 | # helper for adam optimizer |
| 29 | # use tensorflow defaults |
| 30 | def adam(cost, params, lr0=1e-2, beta1=0.9, beta2=0.999, eps=1e-8): |
| 31 | grads = T.grad(cost, params) |
| 32 | updates = [] |
| 33 | time = theano.shared(0) |
| 34 | new_time = time + 1 |
| 35 | updates.append((time, new_time)) |
| 36 | lr = lr0*T.sqrt(1 - beta2**new_time) / (1 - beta1**new_time) |
| 37 | for p, g in zip(params, grads): |
| 38 | m = theano.shared(p.get_value() * 0.) |
| 39 | v = theano.shared(p.get_value() * 0.) |
| 40 | new_m = beta1*m + (1 - beta1)*g |
| 41 | new_v = beta2*v + (1 - beta2)*g*g |
| 42 | new_p = p - lr*new_m / (T.sqrt(new_v) + eps) |
| 43 | updates.append((m, new_m)) |
| 44 | updates.append((v, new_v)) |
| 45 | updates.append((p, new_p)) |
| 46 | return updates |
| 47 | |
| 48 | |
| 49 | # a version of HiddenLayer that keeps track of params |