| 72 | return updates |
| 73 | |
| 74 | class Momentum(Update): |
| 75 | |
| 76 | def __init__(self, lr=0.01, momentum=0.9, *args, **kwargs): |
| 77 | Update.__init__(self, *args, **kwargs) |
| 78 | self.__dict__.update(locals()) |
| 79 | |
| 80 | def __call__(self, params, cost): |
| 81 | updates = [] |
| 82 | grads = T.grad(cost, params) |
| 83 | grads = clip_norms(grads, self.clipnorm) |
| 84 | for p,g in zip(params,grads): |
| 85 | g = self.regularizer.gradient_regularize(p, g) |
| 86 | m = theano.shared(p.get_value() * 0.) |
| 87 | v = (self.momentum * m) - (self.lr * g) |
| 88 | updates.append((m, v)) |
| 89 | |
| 90 | updated_p = p + v |
| 91 | updated_p = self.regularizer.weight_regularize(updated_p) |
| 92 | updates.append((p, updated_p)) |
| 93 | return updates |
| 94 | |
| 95 | |
| 96 | class NAG(Update): |
nothing calls this directly
no outgoing calls
no test coverage detected