Put attention masks on hidden using hidden_features and query.
(query)
| 633 | state = initial_state |
| 634 | |
| 635 | def attention(query): |
| 636 | """Put attention masks on hidden using hidden_features and query.""" |
| 637 | ds = [] # Results of attention reads will be stored here. |
| 638 | if nest.is_sequence(query): # If the query is a tuple, flatten it. |
| 639 | query_list = nest.flatten(query) |
| 640 | for q in query_list: # Check that ndims == 2 if specified. |
| 641 | ndims = q.get_shape().ndims |
| 642 | if ndims: |
| 643 | assert ndims == 2 |
| 644 | query = array_ops.concat(query_list, 1) |
| 645 | for a in xrange(num_heads): |
| 646 | with variable_scope.variable_scope("Attention_%d" % a): |
| 647 | y = Linear(query, attention_vec_size, True)(query) |
| 648 | y = array_ops.reshape(y, [-1, 1, 1, attention_vec_size]) |
| 649 | y = math_ops.cast(y, dtype) |
| 650 | # Attention mask is a softmax of v^T * tanh(...). |
| 651 | s = math_ops.reduce_sum(v[a] * math_ops.tanh(hidden_features[a] + y), |
| 652 | [2, 3]) |
| 653 | a = nn_ops.softmax(math_ops.cast(s, dtype=dtypes.float32)) |
| 654 | # Now calculate the attention-weighted vector d. |
| 655 | a = math_ops.cast(a, dtype) |
| 656 | d = math_ops.reduce_sum( |
| 657 | array_ops.reshape(a, [-1, attn_length, 1, 1]) * hidden, [1, 2]) |
| 658 | ds.append(array_ops.reshape(d, [-1, attn_size])) |
| 659 | return ds |
| 660 | |
| 661 | outputs = [] |
| 662 | prev = None |