| 17 | |
| 18 | |
| 19 | class SimpleRNN: |
| 20 | def __init__(self, D, M, V): |
| 21 | self.D = D # dimensionality of word embedding |
| 22 | self.M = M # hidden layer size |
| 23 | self.V = V # vocabulary size |
| 24 | |
| 25 | def fit(self, X, learning_rate=1., mu=0.99, reg=1.0, activation=T.tanh, epochs=500, show_fig=False): |
| 26 | N = len(X) |
| 27 | D = self.D |
| 28 | M = self.M |
| 29 | V = self.V |
| 30 | self.f = activation |
| 31 | |
| 32 | # initial weights |
| 33 | We = init_weight(V, D) |
| 34 | Wx = init_weight(D, M) |
| 35 | Wh = init_weight(M, M) |
| 36 | bh = np.zeros(M) |
| 37 | h0 = np.zeros(M) |
| 38 | Wo = init_weight(M, V) |
| 39 | bo = np.zeros(V) |
| 40 | |
| 41 | # make them theano shared |
| 42 | self.We = theano.shared(We) |
| 43 | self.Wx = theano.shared(Wx) |
| 44 | self.Wh = theano.shared(Wh) |
| 45 | self.bh = theano.shared(bh) |
| 46 | self.h0 = theano.shared(h0) |
| 47 | self.Wo = theano.shared(Wo) |
| 48 | self.bo = theano.shared(bo) |
| 49 | self.params = [self.We, self.Wx, self.Wh, self.bh, self.h0, self.Wo, self.bo] |
| 50 | |
| 51 | thX = T.ivector('X') |
| 52 | Ei = self.We[thX] # will be a TxD matrix |
| 53 | thY = T.ivector('Y') |
| 54 | |
| 55 | # sentence input: |
| 56 | # [START, w1, w2, ..., wn] |
| 57 | # sentence target: |
| 58 | # [w1, w2, w3, ..., END] |
| 59 | |
| 60 | def recurrence(x_t, h_t1): |
| 61 | # returns h(t), y(t) |
| 62 | h_t = self.f(x_t.dot(self.Wx) + h_t1.dot(self.Wh) + self.bh) |
| 63 | y_t = T.nnet.softmax(h_t.dot(self.Wo) + self.bo) |
| 64 | return h_t, y_t |
| 65 | |
| 66 | [h, y], _ = theano.scan( |
| 67 | fn=recurrence, |
| 68 | outputs_info=[self.h0, None], |
| 69 | sequences=Ei, |
| 70 | n_steps=Ei.shape[0], |
| 71 | ) |
| 72 | |
| 73 | py_x = y[:, 0, :] |
| 74 | prediction = T.argmax(py_x, axis=1) |
| 75 | |
| 76 | cost = -T.mean(T.log(py_x[T.arange(thY.shape[0]), thY])) |
no outgoing calls
no test coverage detected