(savedir)
| 104 | |
| 105 | |
| 106 | def train_model(savedir): |
| 107 | # get the data |
| 108 | sentences, word2idx = get_wiki() #get_text8() |
| 109 | |
| 110 | |
| 111 | # number of unique words |
| 112 | vocab_size = len(word2idx) |
| 113 | |
| 114 | |
| 115 | # config |
| 116 | window_size = 10 |
| 117 | learning_rate = 0.025 |
| 118 | final_learning_rate = 0.0001 |
| 119 | num_negatives = 5 # number of negative samples to draw per input word |
| 120 | samples_per_epoch = int(1e5) |
| 121 | epochs = 20 |
| 122 | D = 50 # word embedding size |
| 123 | |
| 124 | # learning rate decay |
| 125 | learning_rate_delta = (learning_rate - final_learning_rate) / epochs |
| 126 | |
| 127 | # distribution for drawing negative samples |
| 128 | p_neg = get_negative_sampling_distribution(sentences) |
| 129 | |
| 130 | |
| 131 | # params |
| 132 | W = np.random.randn(vocab_size, D).astype(np.float32) # input-to-hidden |
| 133 | V = np.random.randn(D, vocab_size).astype(np.float32) # hidden-to-output |
| 134 | |
| 135 | |
| 136 | # create the model |
| 137 | tf_input = tf.compat.v1.placeholder(tf.int32, shape=(None,)) |
| 138 | tf_negword = tf.compat.v1.placeholder(tf.int32, shape=(None,)) |
| 139 | tf_context = tf.compat.v1.placeholder(tf.int32, shape=(None,)) # targets (context) |
| 140 | tfW = tf.Variable(W) |
| 141 | tfV = tf.Variable(V.T) |
| 142 | # biases = tf.Variable(np.zeros(vocab_size, dtype=np.float32)) |
| 143 | |
| 144 | def dot(A, B): |
| 145 | C = A * B |
| 146 | return tf.reduce_sum(input_tensor=C, axis=1) |
| 147 | |
| 148 | # correct middle word output |
| 149 | emb_input = tf.nn.embedding_lookup(params=tfW, ids=tf_input) # 1 x D |
| 150 | emb_output = tf.nn.embedding_lookup(params=tfV, ids=tf_context) # N x D |
| 151 | correct_output = dot(emb_input, emb_output) # N |
| 152 | # emb_input = tf.transpose(emb_input, (1, 0)) |
| 153 | # correct_output = tf.matmul(emb_output, emb_input) |
| 154 | pos_loss = tf.nn.sigmoid_cross_entropy_with_logits( |
| 155 | labels=tf.ones(tf.shape(input=correct_output)), logits=correct_output) |
| 156 | |
| 157 | # incorrect middle word output |
| 158 | emb_input = tf.nn.embedding_lookup(params=tfW, ids=tf_negword) |
| 159 | incorrect_output = dot(emb_input, emb_output) |
| 160 | # emb_input = tf.transpose(emb_input, (1, 0)) |
| 161 | # incorrect_output = tf.matmul(emb_output, emb_input) |
| 162 | neg_loss = tf.nn.sigmoid_cross_entropy_with_logits( |
| 163 | labels=tf.zeros(tf.shape(input=incorrect_output)), logits=incorrect_output) |
no test coverage detected