| 319 | |
| 320 | |
| 321 | def din_fcn_attention(query, facts, attention_size, mask, stag='null', mode='SUM', softmax_stag=1, time_major=False, return_alphas=False, forCnn=False): |
| 322 | if isinstance(facts, tuple): |
| 323 | # In case of Bi-RNN, concatenate the forward and the backward RNN outputs. |
| 324 | facts = tf.concat(facts, 2) |
| 325 | if len(facts.get_shape().as_list()) == 2: |
| 326 | facts = tf.expand_dims(facts, 1) |
| 327 | |
| 328 | if time_major: |
| 329 | # (T,B,D) => (B,T,D) |
| 330 | facts = tf.array_ops.transpose(facts, [1, 0, 2]) |
| 331 | # Trainable parameters |
| 332 | facts_size = facts.get_shape().as_list()[-1] # D value - hidden size of the RNN layer |
| 333 | querry_size = query.get_shape().as_list()[-1] |
| 334 | query = tf.layers.dense(query, facts_size, activation=None, name='f1' + stag) |
| 335 | query = prelu(query) |
| 336 | queries = tf.tile(query, [1, tf.shape(facts)[1]]) |
| 337 | queries = tf.reshape(queries, tf.shape(facts)) |
| 338 | din_all = tf.concat([queries, facts, queries-facts, queries*facts], axis=-1) |
| 339 | d_layer_1_all = tf.layers.dense(din_all, 80, activation=tf.nn.sigmoid, name='f1_att' + stag) |
| 340 | d_layer_2_all = tf.layers.dense(d_layer_1_all, 40, activation=tf.nn.sigmoid, name='f2_att' + stag) |
| 341 | d_layer_3_all = tf.layers.dense(d_layer_2_all, 1, activation=None, name='f3_att' + stag) |
| 342 | d_layer_3_all = tf.reshape(d_layer_3_all, [-1, 1, tf.shape(facts)[1]]) |
| 343 | scores = d_layer_3_all |
| 344 | # Mask |
| 345 | if mask is not None: |
| 346 | # key_masks = tf.sequence_mask(facts_length, tf.shape(facts)[1]) # [B, T] |
| 347 | key_masks = tf.expand_dims(mask, 1) # [B, 1, T] |
| 348 | paddings = tf.ones_like(scores) * (-2 ** 32 + 1) |
| 349 | if not forCnn: |
| 350 | scores = tf.where(key_masks, scores, paddings) # [B, 1, T] |
| 351 | |
| 352 | # Scale |
| 353 | # scores = scores / (facts.get_shape().as_list()[-1] ** 0.5) |
| 354 | |
| 355 | # Activation |
| 356 | if softmax_stag: |
| 357 | scores = tf.nn.softmax(scores) # [B, 1, T] |
| 358 | |
| 359 | # Weighted sum |
| 360 | if mode == 'SUM': |
| 361 | output = tf.matmul(scores, facts) # [B, 1, H] |
| 362 | # output = tf.reshape(output, [-1, tf.shape(facts)[-1]]) |
| 363 | else: |
| 364 | scores = tf.reshape(scores, [-1, tf.shape(facts)[1]]) |
| 365 | output = facts * tf.expand_dims(scores, -1) |
| 366 | output = tf.reshape(output, tf.shape(facts)) |
| 367 | if return_alphas: |
| 368 | return output, scores |
| 369 | return output |
| 370 | |
| 371 | def self_attention(facts, ATTENTION_SIZE, mask, stag='null'): |
| 372 | if len(facts.get_shape().as_list()) == 2: |