| 39 | |
| 40 | |
| 41 | class RNTN: |
| 42 | def __init__(self, V, D, K, activation): |
| 43 | self.D = D |
| 44 | self.f = activation |
| 45 | |
| 46 | # word embedding |
| 47 | We = init_weight(V, D) |
| 48 | |
| 49 | # quadratic terms |
| 50 | W11 = np.random.randn(D, D, D) / np.sqrt(3*D) |
| 51 | W22 = np.random.randn(D, D, D) / np.sqrt(3*D) |
| 52 | W12 = np.random.randn(D, D, D) / np.sqrt(3*D) |
| 53 | |
| 54 | # linear terms |
| 55 | W1 = init_weight(D, D) |
| 56 | W2 = init_weight(D, D) |
| 57 | |
| 58 | # bias |
| 59 | bh = np.zeros(D) |
| 60 | |
| 61 | # output layer |
| 62 | Wo = init_weight(D, K) |
| 63 | bo = np.zeros(K) |
| 64 | |
| 65 | # make them tensorflow variables |
| 66 | self.We = tf.Variable(We.astype(np.float32)) |
| 67 | self.W11 = tf.Variable(W11.astype(np.float32)) |
| 68 | self.W22 = tf.Variable(W22.astype(np.float32)) |
| 69 | self.W12 = tf.Variable(W12.astype(np.float32)) |
| 70 | self.W1 = tf.Variable(W1.astype(np.float32)) |
| 71 | self.W2 = tf.Variable(W2.astype(np.float32)) |
| 72 | self.bh = tf.Variable(bh.astype(np.float32)) |
| 73 | self.Wo = tf.Variable(Wo.astype(np.float32)) |
| 74 | self.bo = tf.Variable(bo.astype(np.float32)) |
| 75 | self.params = [self.We, self.W11, self.W22, self.W12, self.W1, self.W2, self.Wo] |
| 76 | |
| 77 | def fit(self, trees, lr=1e-2, mu=0.9, reg=1e-1, epochs=5): |
| 78 | train_ops = [] |
| 79 | costs = [] |
| 80 | predictions = [] |
| 81 | all_labels = [] |
| 82 | i = 0 |
| 83 | N = len(trees) |
| 84 | print("Compiling ops") |
| 85 | for t in trees: |
| 86 | i += 1 |
| 87 | sys.stdout.write("%d/%d\r" % (i, N)) |
| 88 | sys.stdout.flush() |
| 89 | logits = self.get_output(t) |
| 90 | labels = get_labels(t) |
| 91 | all_labels.append(labels) |
| 92 | |
| 93 | cost = self.get_cost(logits, labels, reg) |
| 94 | costs.append(cost) |
| 95 | |
| 96 | prediction = tf.argmax(logits, 1) |
| 97 | predictions.append(prediction) |
| 98 | |