| 11 | from a2_base_model import BaseClass |
| 12 | import time |
| 13 | class Encoder(BaseClass): |
| 14 | def __init__(self,d_model,d_k,d_v,sequence_length,h,batch_size,num_layer,Q,K_s,type='encoder',mask=None,dropout_keep_prob=None,use_residual_conn=True): |
| 15 | """ |
| 16 | :param d_model: |
| 17 | :param d_k: |
| 18 | :param d_v: |
| 19 | :param sequence_length: |
| 20 | :param h: |
| 21 | :param batch_size: |
| 22 | :param embedded_words: shape:[batch_size*sequence_length,embed_size] |
| 23 | """ |
| 24 | super(Encoder, self).__init__(d_model,d_k,d_v,sequence_length,h,batch_size,num_layer=num_layer) |
| 25 | self.Q=Q |
| 26 | self.K_s=K_s |
| 27 | self.type=type |
| 28 | self.mask=mask |
| 29 | self.initializer = tf.random_normal_initializer(stddev=0.1) |
| 30 | self.dropout_keep_prob=dropout_keep_prob |
| 31 | self.use_residual_conn=use_residual_conn |
| 32 | |
| 33 | def encoder_fn(self): |
| 34 | start = time.time() |
| 35 | print("encoder_fn.started.") |
| 36 | Q=self.Q |
| 37 | K_s=self.K_s |
| 38 | for layer_index in range(self.num_layer): |
| 39 | Q, K_s=self.encoder_single_layer(Q,K_s,layer_index) |
| 40 | print("encoder_fn.",layer_index,".Q:",Q,";K_s:",K_s) |
| 41 | end = time.time() |
| 42 | print("encoder_fn.ended.Q:",Q,";K_s:",K_s,";time spent:",(end-start)) |
| 43 | return Q,K_s |
| 44 | |
| 45 | def encoder_single_layer(self,Q,K_s,layer_index): |
| 46 | """ |
| 47 | singel layer for encoder.each layers has two sub-layers: |
| 48 | the first is multi-head self-attention mechanism; the second is position-wise fully connected feed-forward network. |
| 49 | for each sublayer. use LayerNorm(x+Sublayer(x)). input and output of last dimension: d_model |
| 50 | :param Q: shape should be: [batch_size*sequence_length,d_model] |
| 51 | :param K_s: shape should be: [batch_size*sequence_length,d_model] |
| 52 | :return:output: shape should be:[batch_size*sequence_length,d_model] |
| 53 | """ |
| 54 | #1.1 the first is multi-head self-attention mechanism |
| 55 | multi_head_attention_output=self.sub_layer_multi_head_attention(layer_index,Q,K_s,self.type,mask=self.mask,dropout_keep_prob=self.dropout_keep_prob) #[batch_size,sequence_length,d_model] |
| 56 | #1.2 use LayerNorm(x+Sublayer(x)). all dimension=512. |
| 57 | multi_head_attention_output=self.sub_layer_layer_norm_residual_connection(K_s ,multi_head_attention_output,layer_index,'encoder_multi_head_attention',dropout_keep_prob=self.dropout_keep_prob,use_residual_conn=self.use_residual_conn) |
| 58 | |
| 59 | #2.1 the second is position-wise fully connected feed-forward network. |
| 60 | postion_wise_feed_forward_output=self.sub_layer_postion_wise_feed_forward(multi_head_attention_output,layer_index,self.type) |
| 61 | #2.2 use LayerNorm(x+Sublayer(x)). all dimension=512. |
| 62 | postion_wise_feed_forward_output= self.sub_layer_layer_norm_residual_connection(multi_head_attention_output,postion_wise_feed_forward_output,layer_index,'encoder_postion_wise_ff',dropout_keep_prob=self.dropout_keep_prob) |
| 63 | return postion_wise_feed_forward_output,postion_wise_feed_forward_output |
| 64 | |
| 65 | |
| 66 | def init(): |