Apply QKV attention. :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. :return: an [N x (H * C) x T] tensor after attention.
(self, qkv)
| 355 | self.n_heads = n_heads |
| 356 | |
| 357 | def forward(self, qkv): |
| 358 | """ |
| 359 | Apply QKV attention. |
| 360 | :param qkv: an [N x (H * 3 * C) x T] tensor of Qs, Ks, and Vs. |
| 361 | :return: an [N x (H * C) x T] tensor after attention. |
| 362 | """ |
| 363 | bs, width, length = qkv.shape |
| 364 | assert width % (3 * self.n_heads) == 0 |
| 365 | ch = width // (3 * self.n_heads) |
| 366 | q, k, v = qkv.reshape(bs * self.n_heads, ch * 3, length).split(ch, dim=1) |
| 367 | scale = 1 / math.sqrt(math.sqrt(ch)) |
| 368 | weight = th.einsum( |
| 369 | "bct,bcs->bts", q * scale, k * scale |
| 370 | ) # More stable with f16 than dividing afterwards |
| 371 | weight = th.softmax(weight.float(), dim=-1).type(weight.dtype) |
| 372 | a = th.einsum("bts,bcs->bct", weight, v) |
| 373 | return a.reshape(bs, -1, length) |
| 374 | |
| 375 | @staticmethod |
| 376 | def count_flops(model, _x, y): |
nothing calls this directly
no outgoing calls
no test coverage detected