| 417 | return self_attention |
| 418 | |
| 419 | def din_fcn_shine(query, facts, attention_size, mask, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False): |
| 420 | if isinstance(facts, tuple): |
| 421 | # In case of Bi-RNN, concatenate the forward and the backward RNN outputs. |
| 422 | facts = tf.concat(facts, 2) |
| 423 | |
| 424 | if time_major: |
| 425 | # (T,B,D) => (B,T,D) |
| 426 | facts = tf.array_ops.transpose(facts, [1, 0, 2]) |
| 427 | # Trainable parameters |
| 428 | mask = tf.equal(mask, tf.ones_like(mask)) |
| 429 | facts_size = facts.get_shape().as_list()[-1] # D value - hidden size of the RNN layer |
| 430 | querry_size = query.get_shape().as_list()[-1] |
| 431 | query = tf.layers.dense(query, facts_size, activation=None, name='f1_trans_shine' + stag) |
| 432 | query = prelu(query) |
| 433 | queries = tf.tile(query, [1, tf.shape(facts)[1]]) |
| 434 | queries = tf.reshape(queries, tf.shape(facts)) |
| 435 | din_all = tf.concat([queries, facts, queries-facts, queries*facts], axis=-1) |
| 436 | d_layer_1_all = tf.layers.dense(din_all, facts_size, activation=tf.nn.sigmoid, name='f1_shine_att' + stag) |
| 437 | d_layer_2_all = tf.layers.dense(d_layer_1_all, facts_size, activation=tf.nn.sigmoid, name='f2_shine_att' + stag) |
| 438 | d_layer_2_all = tf.reshape(d_layer_2_all, tf.shape(facts)) |
| 439 | output = d_layer_2_all |
| 440 | return output |
| 441 | |