Method decoder_single_layer

a07_Transformer/a2_decoder.py:54–89

singel layer for decoder. each layers has three sub-layers: the first is multi-head self-attention(mask) mechanism; the second is multi-head attention over output of encoder the third is position-wise fully connected feed-forward network. for each sublay

(self,Q,K_s,layer_index)

        return Q,K_s

    def decoder_single_layer(self,Q,K_s,layer_index):
        """
        singel layer for decoder. each layers has three sub-layers:
        the first is multi-head self-attention(mask) mechanism;
        the second is multi-head attention over output of encoder
        the third is position-wise fully connected feed-forward network.
        for each sublayer. use LayerNorm(x+Sublayer(x)). input and output of last dimension: d_model=512s.
        :param Q: shape should be:       [batch_size,sequence_length,d_model]
        :param K_s: shape should be:     [batch_size,sequence_length,d_model]
        :param layer_index: index of layer
        :param mask: mask is a list. length is sequence_length. each element is a scaler value. e.g. [1,1,1,-1000000,-1000000,-1000000,....-1000000]
        :return:output: shape should be:[batch_size*sequence_length,d_model]
        """
        print("#decoder#decoder_single_layer",layer_index,"====================================>")
        # 1.1 the first is masked multi-head self-attention mechanism
        multi_head_attention_output=self.sub_layer_multi_head_attention(layer_index,Q,K_s,self.type,is_training=self.is_training,mask=self.mask,dropout_keep_prob=self.dropout_keep_prob) #[batch_size*sequence_length,d_model]
        #1.2 use LayerNorm(x+Sublayer(x)). all dimension=512.
        multi_head_attention_output=self.sub_layer_layer_norm_residual_connection(K_s,multi_head_attention_output,layer_index,'decoder_multi_head_attention',dropout_keep_prob=self.dropout_keep_prob)


        # 2.1 the second is multi-head attention over output of encoder
        # IMPORTANT!!! check two parameters below: Q: should from decoder; K_s: should be the output of encoder
        attention_enc_dec=AttentionEncoderDecoder(self.d_model,self.d_k,self.d_v,self.sequence_length,self.h,self.batch_size,
                                                  multi_head_attention_output,self.K_v_encoder,layer_index,self.decoder_sent_length,dropout_keep_prob=self.dropout_keep_prob)
        attention_enc_dec_output=attention_enc_dec.attention_encoder_decoder_fn()
        print("decoder.2.1.attention_enc_dec_output:",attention_enc_dec_output)
        # 2.2 use LayerNorm(x+Sublayer(x)). all dimension=512.
        attention_enc_dec_output=self.sub_layer_layer_norm_residual_connection(multi_head_attention_output,attention_enc_dec_output, layer_index,
                                                                               'decoder_attention_encoder_decoder',dropout_keep_prob=self.dropout_keep_prob)

        # 3.1 the second is position-wise fully connected feed-forward network.
        postion_wise_feed_forward_output=self.sub_layer_postion_wise_feed_forward(attention_enc_dec_output,layer_index,self.type)
        print("decoder.3.1.postion_wise_feed_forward_output:",postion_wise_feed_forward_output)
        #3.2 use LayerNorm(x+Sublayer(x)). all dimension=512.
        postion_wise_feed_forward_output=self.sub_layer_layer_norm_residual_connection(attention_enc_dec_output,postion_wise_feed_forward_output, layer_index, 'decoder_position_ff',dropout_keep_prob=self.dropout_keep_prob)
        return postion_wise_feed_forward_output,postion_wise_feed_forward_output

#####################BELOW IS TEST METHOD FOR DECODER: FROM SMALL FUNCTION  TO WHOLE FUNCTION OF DECODER：################################################
#test decoder for single layer