| 30 | self.hidden_layer_sizes = hidden_layer_sizes |
| 31 | |
| 32 | def fit(self, X, Y, activation=T.tanh, learning_rate=1e-1, mu=0.5, reg=0, epochs=2000, show_fig=False): |
| 33 | N, t, D = X.shape |
| 34 | |
| 35 | self.hidden_layers = [] |
| 36 | Mi = D |
| 37 | for Mo in self.hidden_layer_sizes: |
| 38 | ru = GRU(Mi, Mo, activation) |
| 39 | self.hidden_layers.append(ru) |
| 40 | Mi = Mo |
| 41 | |
| 42 | Wo = np.random.randn(Mi) / np.sqrt(Mi) |
| 43 | bo = 0.0 |
| 44 | self.Wo = theano.shared(Wo) |
| 45 | self.bo = theano.shared(bo) |
| 46 | self.params = [self.Wo, self.bo] |
| 47 | for ru in self.hidden_layers: |
| 48 | self.params += ru.params |
| 49 | |
| 50 | lr = T.scalar('lr') |
| 51 | thX = T.matrix('X') |
| 52 | thY = T.scalar('Y') |
| 53 | Yhat = self.forward(thX)[-1] |
| 54 | |
| 55 | # let's return py_x too so we can draw a sample instead |
| 56 | self.predict_op = theano.function( |
| 57 | inputs=[thX], |
| 58 | outputs=Yhat, |
| 59 | allow_input_downcast=True, |
| 60 | ) |
| 61 | |
| 62 | cost = T.mean((thY - Yhat)*(thY - Yhat)) |
| 63 | grads = T.grad(cost, self.params) |
| 64 | dparams = [theano.shared(p.get_value()*0) for p in self.params] |
| 65 | |
| 66 | updates = [ |
| 67 | (p, p + mu*dp - lr*g) for p, dp, g in zip(self.params, dparams, grads) |
| 68 | ] + [ |
| 69 | (dp, mu*dp - lr*g) for dp, g in zip(dparams, grads) |
| 70 | ] |
| 71 | |
| 72 | self.train_op = theano.function( |
| 73 | inputs=[lr, thX, thY], |
| 74 | outputs=cost, |
| 75 | updates=updates |
| 76 | ) |
| 77 | |
| 78 | costs = [] |
| 79 | for i in xrange(epochs): |
| 80 | t0 = datetime.now() |
| 81 | X, Y = shuffle(X, Y) |
| 82 | n_correct = 0 |
| 83 | n_total = 0 |
| 84 | cost = 0 |
| 85 | for j in xrange(N): |
| 86 | |
| 87 | c = self.train_op(learning_rate, X[j], Y[j]) |
| 88 | cost += c |
| 89 | if i % 10 == 0: |