elman neural net model
| 137 | |
| 138 | # start-snippet-2 |
| 139 | class RNNSLU(object): |
| 140 | ''' elman neural net model ''' |
| 141 | def __init__(self, nh, nc, ne, de, cs): |
| 142 | ''' |
| 143 | nh :: dimension of the hidden layer |
| 144 | nc :: number of classes |
| 145 | ne :: number of word embeddings in the vocabulary |
| 146 | de :: dimension of the word embeddings |
| 147 | cs :: word window context size |
| 148 | ''' |
| 149 | # parameters of the model |
| 150 | self.emb = theano.shared(name='embeddings', |
| 151 | value=0.2 * numpy.random.uniform(-1.0, 1.0, |
| 152 | (ne+1, de)) |
| 153 | # add one for padding at the end |
| 154 | .astype(theano.config.floatX)) |
| 155 | self.wx = theano.shared(name='wx', |
| 156 | value=0.2 * numpy.random.uniform(-1.0, 1.0, |
| 157 | (de * cs, nh)) |
| 158 | .astype(theano.config.floatX)) |
| 159 | self.wh = theano.shared(name='wh', |
| 160 | value=0.2 * numpy.random.uniform(-1.0, 1.0, |
| 161 | (nh, nh)) |
| 162 | .astype(theano.config.floatX)) |
| 163 | self.w = theano.shared(name='w', |
| 164 | value=0.2 * numpy.random.uniform(-1.0, 1.0, |
| 165 | (nh, nc)) |
| 166 | .astype(theano.config.floatX)) |
| 167 | self.bh = theano.shared(name='bh', |
| 168 | value=numpy.zeros(nh, |
| 169 | dtype=theano.config.floatX)) |
| 170 | self.b = theano.shared(name='b', |
| 171 | value=numpy.zeros(nc, |
| 172 | dtype=theano.config.floatX)) |
| 173 | self.h0 = theano.shared(name='h0', |
| 174 | value=numpy.zeros(nh, |
| 175 | dtype=theano.config.floatX)) |
| 176 | |
| 177 | # bundle |
| 178 | self.params = [self.emb, self.wx, self.wh, self.w, |
| 179 | self.bh, self.b, self.h0] |
| 180 | # end-snippet-2 |
| 181 | # as many columns as context window size |
| 182 | # as many lines as words in the sentence |
| 183 | # start-snippet-3 |
| 184 | idxs = T.imatrix() |
| 185 | x = self.emb[idxs].reshape((idxs.shape[0], de*cs)) |
| 186 | y_sentence = T.ivector('y_sentence') # labels |
| 187 | # end-snippet-3 start-snippet-4 |
| 188 | |
| 189 | def recurrence(x_t, h_tm1): |
| 190 | h_t = T.nnet.sigmoid(T.dot(x_t, self.wx) |
| 191 | + T.dot(h_tm1, self.wh) + self.bh) |
| 192 | s_t = T.nnet.softmax(T.dot(h_t, self.w) + self.b) |
| 193 | return [h_t, s_t] |
| 194 | |
| 195 | [h, s], _ = theano.scan(fn=recurrence, |
| 196 | sequences=x, |