| 8 | #from a07_Transformer.a2_multi_head_attention import MultiHeadAttention |
| 9 | |
| 10 | class EntityNetwork: |
| 11 | def __init__(self, num_classes, learning_rate, batch_size, decay_steps, decay_rate, sequence_length, story_length, |
| 12 | vocab_size, embed_size,hidden_size, is_training, multi_label_flag=False,block_size=20, |
| 13 | initializer=tf.random_normal_initializer(stddev=0.1),clip_gradients=5.0,use_bi_lstm=False,use_additive_attention=False):#0.01 |
| 14 | """init all hyperparameter here""" |
| 15 | # set hyperparamter |
| 16 | self.num_classes = num_classes |
| 17 | self.batch_size = batch_size |
| 18 | self.sequence_length = sequence_length |
| 19 | self.vocab_size = vocab_size |
| 20 | self.embed_size = embed_size |
| 21 | self.is_training = is_training |
| 22 | self.learning_rate = tf.Variable(learning_rate, trainable=False, name="learning_rate")#TODO ADD learning_rate |
| 23 | self.learning_rate_decay_half_op = tf.assign(self.learning_rate, self.learning_rate * 0.5) |
| 24 | self.initializer = initializer |
| 25 | self.multi_label_flag = multi_label_flag |
| 26 | self.hidden_size = hidden_size |
| 27 | self.clip_gradients=clip_gradients |
| 28 | self.story_length=story_length |
| 29 | self.block_size=block_size |
| 30 | self.use_bi_lstm=use_bi_lstm |
| 31 | self.dimension=self.hidden_size*2 if self.use_bi_lstm else self.hidden_size #if use bi-lstm, set dimension value, so it can be used later for parameter. |
| 32 | self.use_additive_attention=use_additive_attention |
| 33 | |
| 34 | # add placeholder (X,label) |
| 35 | # self.input_x = tf.placeholder(tf.int32, [None, self.num_sentences,self.sequence_length], name="input_x") # X |
| 36 | self.story=tf.placeholder(tf.int32,[None,self.story_length,self.sequence_length],name="story") |
| 37 | self.query = tf.placeholder(tf.int32, [None, self.sequence_length], name="question") |
| 38 | |
| 39 | self.answer_single = tf.placeholder(tf.int32, [None,], name="input_y") # y:[None,num_classes] |
| 40 | self.answer_multilabel = tf.placeholder(tf.float32, [None, self.num_classes],name="input_y_multilabel") # y:[None,num_classes]. this is for multi-label classification only. |
| 41 | self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob") |
| 42 | |
| 43 | self.global_step = tf.Variable(0, trainable=False, name="Global_Step") |
| 44 | self.epoch_step = tf.Variable(0, trainable=False, name="Epoch_Step") |
| 45 | self.epoch_increment = tf.assign(self.epoch_step, tf.add(self.epoch_step, tf.constant(1))) |
| 46 | self.decay_steps, self.decay_rate = decay_steps, decay_rate |
| 47 | |
| 48 | self.instantiate_weights() |
| 49 | self.logits = self.inference() # [None, self.label_size]. main computation graph is here. |
| 50 | |
| 51 | self.predictions = tf.argmax(self.logits, 1, name="predictions") # shape:[None,] |
| 52 | if not self.multi_label_flag: |
| 53 | correct_prediction = tf.equal(tf.cast(self.predictions, tf.int32),self.answer_single) # tf.argmax(self.logits, 1)-->[batch_size] |
| 54 | self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name="Accuracy") # shape=() |
| 55 | else: |
| 56 | self.accuracy = tf.constant(0.5) # fuke accuracy. (you can calcuate accuracy outside of graph using method calculate_accuracy(...) in train.py) |
| 57 | |
| 58 | if not is_training: |
| 59 | return |
| 60 | if multi_label_flag: |
| 61 | print("going to use multi label loss.") |
| 62 | self.loss_val = self.loss_multilabel() |
| 63 | else: |
| 64 | print("going to use single label loss.") |
| 65 | self.loss_val = self.loss() |
| 66 | self.train_op = self.train() |
| 67 | |