| 41 | |
| 42 | # helper for adam optimizer |
| 43 | def adam(params, grads): |
| 44 | updates = [] |
| 45 | time = theano.shared(0) |
| 46 | new_time = time + 1 |
| 47 | updates.append((time, new_time)) |
| 48 | lr = LEARNING_RATE*T.sqrt(1 - BETA2**new_time) / (1 - BETA1**new_time) |
| 49 | for p, g in zip(params, grads): |
| 50 | m = theano.shared(p.get_value() * 0.) |
| 51 | v = theano.shared(p.get_value() * 0.) |
| 52 | new_m = BETA1*m + (1 - BETA1)*g |
| 53 | new_v = BETA2*v + (1 - BETA2)*g*g |
| 54 | new_p = p - lr*new_m / (T.sqrt(new_v) + EPSILON) |
| 55 | updates.append((m, new_m)) |
| 56 | updates.append((v, new_v)) |
| 57 | updates.append((p, new_p)) |
| 58 | return updates |
| 59 | |
| 60 | |
| 61 | # helper for batch norm |