| 85 | return res |
| 86 | |
| 87 | def din_attention(query, facts, attention_size, mask=None, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False): |
| 88 | if isinstance(facts, tuple): |
| 89 | # In case of Bi-RNN, concatenate the forward and the backward RNN outputs. |
| 90 | facts = tf.concat(facts, 2) |
| 91 | print ("query_size mismatch") |
| 92 | query = tf.concat(values = [ |
| 93 | query, |
| 94 | query, |
| 95 | ], axis=1) |
| 96 | |
| 97 | if time_major: |
| 98 | # (T,B,D) => (B,T,D) |
| 99 | facts = tf.array_ops.transpose(facts, [1, 0, 2]) |
| 100 | facts_size = facts.get_shape().as_list()[-1] # D value - hidden size of the RNN layer |
| 101 | querry_size = query.get_shape().as_list()[-1] |
| 102 | queries = tf.tile(query, [1, tf.shape(facts)[1]]) |
| 103 | queries = tf.reshape(queries, tf.shape(facts)) |
| 104 | din_all = tf.concat([queries, facts, queries-facts, queries*facts], axis=-1) |
| 105 | d_layer_1_all = tf.layers.dense(din_all, 80, activation=tf.nn.sigmoid, name='f1_att' + stag) |
| 106 | d_layer_2_all = tf.layers.dense(d_layer_1_all, 40, activation=tf.nn.sigmoid, name='f2_att' + stag) |
| 107 | d_layer_3_all = tf.layers.dense(d_layer_2_all, 1, activation=None, name='f3_att' + stag) |
| 108 | d_layer_3_all = tf.reshape(d_layer_3_all, [-1, 1, tf.shape(facts)[1]]) |
| 109 | scores = d_layer_3_all |
| 110 | |
| 111 | if mask is not None: |
| 112 | mask = tf.equal(mask, tf.ones_like(mask)) |
| 113 | key_masks = tf.expand_dims(mask, 1) # [B, 1, T] |
| 114 | paddings = tf.ones_like(scores) * (-2 ** 32 + 1) |
| 115 | scores = tf.where(key_masks, scores, paddings) # [B, 1, T] |
| 116 | |
| 117 | # Activation |
| 118 | if softmax_stag: |
| 119 | scores = tf.nn.softmax(scores) # [B, 1, T] |
| 120 | |
| 121 | # Weighted sum |
| 122 | if mode == 'SUM': |
| 123 | output = tf.matmul(scores, facts) # [B, 1, H] |
| 124 | # output = tf.reshape(output, [-1, tf.shape(facts)[-1]]) |
| 125 | else: |
| 126 | scores = tf.reshape(scores, [-1, tf.shape(facts)[1]]) |
| 127 | output = facts * tf.expand_dims(scores, -1) |
| 128 | output = tf.reshape(output, tf.shape(facts)) |
| 129 | |
| 130 | if return_alphas: |
| 131 | return output, scores |
| 132 | |
| 133 | return output |
| 134 | |
| 135 | |
| 136 | class VecAttGRUCell(RNNCell): |