| 11 | return x |
| 12 | |
| 13 | class PerceiverAttention(nn.Module): |
| 14 | def __init__(self, *, dim, dim_head=64, heads=8): |
| 15 | super().__init__() |
| 16 | self.scale = dim_head**-0.5 |
| 17 | self.dim_head = dim_head |
| 18 | self.heads = heads |
| 19 | inner_dim = dim_head * heads |
| 20 | |
| 21 | self.norm1 = nn.LayerNorm(dim) |
| 22 | self.norm2 = nn.LayerNorm(dim) |
| 23 | |
| 24 | self.to_q = nn.Linear(dim, inner_dim, bias=False) |
| 25 | self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False) |
| 26 | self.to_out = nn.Linear(inner_dim, dim, bias=False) |
| 27 | |
| 28 | def forward(self, x, latents): |
| 29 | """ |
| 30 | Args: |
| 31 | x (torch.Tensor): image features |
| 32 | shape (b, n1, D) |
| 33 | latent (torch.Tensor): latent features |
| 34 | shape (b, n2, D) |
| 35 | """ |
| 36 | |
| 37 | x = self.norm1(x) |
| 38 | latents = self.norm2(latents) |
| 39 | |
| 40 | b, l, _ = latents.shape |
| 41 | |
| 42 | q = self.to_q(latents) |
| 43 | kv_input = torch.cat((x, latents), dim=-2) |
| 44 | k, v = self.to_kv(kv_input).chunk(2, dim=-1) |
| 45 | |
| 46 | q = reshape_tensor(q, self.heads) |
| 47 | k = reshape_tensor(k, self.heads) |
| 48 | v = reshape_tensor(v, self.heads) |
| 49 | |
| 50 | # attention |
| 51 | scale = 1 / math.sqrt(math.sqrt(self.dim_head)) |
| 52 | weight = (q * scale) @ (k * scale).transpose(-2, -1) |
| 53 | weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) |
| 54 | out = weight @ v |
| 55 | |
| 56 | out = out.permute(0, 2, 1, 3).reshape(b, l, -1) |
| 57 | |
| 58 | return self.to_out(out) |
| 59 | |
| 60 | def FeedForward(dim, mult=4): |
| 61 | inner_dim = int(dim * mult) |