| 16 | import time |
| 17 | |
| 18 | class Decoder(BaseClass): |
| 19 | def __init__(self,d_model,d_k,d_v,sequence_length,h,batch_size,Q,K_s,K_v_encoder,decoder_sent_length, |
| 20 | num_layer=6,type='decoder',is_training=True,mask=None,dropout_keep_prob=None): |
| 21 | """ |
| 22 | :param d_model: |
| 23 | :param d_k: |
| 24 | :param d_v: |
| 25 | :param sequence_length: |
| 26 | :param h: |
| 27 | :param batch_size: |
| 28 | :param Q: |
| 29 | :param K_s: |
| 30 | :param K_v_encoder: shape:[batch_size,sequence_length,embed_size]. it is the output from encoder |
| 31 | """ |
| 32 | super(Decoder, self).__init__(d_model, d_k, d_v, sequence_length, h, batch_size, num_layer=num_layer) |
| 33 | self.Q=Q |
| 34 | self.K_s=K_s |
| 35 | self.K_v_encoder=K_v_encoder |
| 36 | self.type=type |
| 37 | self.initializer = tf.random_normal_initializer(stddev=0.1) |
| 38 | self.is_training=is_training |
| 39 | self.decoder_sent_length=decoder_sent_length |
| 40 | self.mask=mask |
| 41 | self.dropout_keep_prob=dropout_keep_prob |
| 42 | |
| 43 | def decoder_fn(self): |
| 44 | start = time.time() |
| 45 | print("decoder.decoder_fn.started.") |
| 46 | Q=self.Q |
| 47 | K_s=self.K_s |
| 48 | for layer_index in range(self.num_layer): |
| 49 | Q,K_s=self.decoder_single_layer(Q, K_s, layer_index) |
| 50 | end = time.time() |
| 51 | print("decoder.decoder_fn.ended.Q:", Q, " ;K_s:", K_s,";time spent:",(end-start)) |
| 52 | return Q,K_s |
| 53 | |
| 54 | def decoder_single_layer(self,Q,K_s,layer_index): |
| 55 | """ |
| 56 | singel layer for decoder. each layers has three sub-layers: |
| 57 | the first is multi-head self-attention(mask) mechanism; |
| 58 | the second is multi-head attention over output of encoder |
| 59 | the third is position-wise fully connected feed-forward network. |
| 60 | for each sublayer. use LayerNorm(x+Sublayer(x)). input and output of last dimension: d_model=512s. |
| 61 | :param Q: shape should be: [batch_size,sequence_length,d_model] |
| 62 | :param K_s: shape should be: [batch_size,sequence_length,d_model] |
| 63 | :param layer_index: index of layer |
| 64 | :param mask: mask is a list. length is sequence_length. each element is a scaler value. e.g. [1,1,1,-1000000,-1000000,-1000000,....-1000000] |
| 65 | :return:output: shape should be:[batch_size*sequence_length,d_model] |
| 66 | """ |
| 67 | print("#decoder#decoder_single_layer",layer_index,"====================================>") |
| 68 | # 1.1 the first is masked multi-head self-attention mechanism |
| 69 | multi_head_attention_output=self.sub_layer_multi_head_attention(layer_index,Q,K_s,self.type,is_training=self.is_training,mask=self.mask,dropout_keep_prob=self.dropout_keep_prob) #[batch_size*sequence_length,d_model] |
| 70 | #1.2 use LayerNorm(x+Sublayer(x)). all dimension=512. |
| 71 | multi_head_attention_output=self.sub_layer_layer_norm_residual_connection(K_s,multi_head_attention_output,layer_index,'decoder_multi_head_attention',dropout_keep_prob=self.dropout_keep_prob) |
| 72 | |
| 73 | |
| 74 | # 2.1 the second is multi-head attention over output of encoder |
| 75 | # IMPORTANT!!! check two parameters below: Q: should from decoder; K_s: should be the output of encoder |