| 236 | attn_dot = Dot(axes=1) # to perform the weighted sum of alpha[t] * h[t] |
| 237 | |
| 238 | def one_step_attention(h, st_1): |
| 239 | # h = h(1), ..., h(Tx), shape = (Tx, LATENT_DIM * 2) |
| 240 | # st_1 = s(t-1), shape = (LATENT_DIM_DECODER,) |
| 241 | |
| 242 | # copy s(t-1) Tx times |
| 243 | # now shape = (Tx, LATENT_DIM_DECODER) |
| 244 | st_1 = attn_repeat_layer(st_1) |
| 245 | |
| 246 | # Concatenate all h(t)'s with s(t-1) |
| 247 | # Now of shape (Tx, LATENT_DIM_DECODER + LATENT_DIM * 2) |
| 248 | x = attn_concat_layer([h, st_1]) |
| 249 | |
| 250 | # Neural net first layer |
| 251 | x = attn_dense1(x) |
| 252 | |
| 253 | # Neural net second layer with special softmax over time |
| 254 | alphas = attn_dense2(x) |
| 255 | |
| 256 | # "Dot" the alphas and the h's |
| 257 | # Remember a.dot(b) = sum over a[t] * b[t] |
| 258 | context = attn_dot([alphas, h]) |
| 259 | |
| 260 | return context |
| 261 | |
| 262 | |
| 263 | # define the rest of the decoder (after attention) |