| 138 | |
| 139 | |
| 140 | class Adam(Update): |
| 141 | |
| 142 | def __init__(self, lr=0.001, b1=0.9, b2=0.999, e=1e-8, l=1-1e-8, *args, **kwargs): |
| 143 | Update.__init__(self, *args, **kwargs) |
| 144 | self.__dict__.update(locals()) |
| 145 | |
| 146 | def __call__(self, params, cost): |
| 147 | updates = [] |
| 148 | grads = T.grad(cost, params) |
| 149 | grads = clip_norms(grads, self.clipnorm) |
| 150 | t = theano.shared(floatX(1.)) |
| 151 | b1_t = self.b1*self.l**(t-1) |
| 152 | |
| 153 | for p, g in zip(params, grads): |
| 154 | g = self.regularizer.gradient_regularize(p, g) |
| 155 | m = theano.shared(p.get_value() * 0.) |
| 156 | v = theano.shared(p.get_value() * 0.) |
| 157 | |
| 158 | m_t = b1_t*m + (1 - b1_t)*g |
| 159 | v_t = self.b2*v + (1 - self.b2)*g**2 |
| 160 | m_c = m_t / (1-self.b1**t) |
| 161 | v_c = v_t / (1-self.b2**t) |
| 162 | p_t = p - (self.lr * m_c) / (T.sqrt(v_c) + self.e) |
| 163 | p_t = self.regularizer.weight_regularize(p_t) |
| 164 | updates.append((m, m_t)) |
| 165 | updates.append((v, v_t)) |
| 166 | updates.append((p, p_t) ) |
| 167 | updates.append((t, t + 1.)) |
| 168 | return updates |
| 169 | |
| 170 | |
| 171 | class Adagrad(Update): |
nothing calls this directly
no outgoing calls
no test coverage detected