| 55 | |
| 56 | class AttentionPool2d(nn.Module): |
| 57 | def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): |
| 58 | super().__init__() |
| 59 | self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5) |
| 60 | self.k_proj = nn.Linear(embed_dim, embed_dim) |
| 61 | self.q_proj = nn.Linear(embed_dim, embed_dim) |
| 62 | self.v_proj = nn.Linear(embed_dim, embed_dim) |
| 63 | self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) |
| 64 | self.num_heads = num_heads |
| 65 | |
| 66 | def forward(self, x): |
| 67 | x = x.reshape(x.shape[0], x.shape[1], x.shape[2] * x.shape[3]).permute(2, 0, 1) # NCHW -> (HW)NC |