| 57 | |
| 58 | class AttentionPool2d(nn.Module): |
| 59 | def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): |
| 60 | super().__init__() |
| 61 | self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5) |
| 62 | self.k_proj = nn.Linear(embed_dim, embed_dim) |
| 63 | self.q_proj = nn.Linear(embed_dim, embed_dim) |
| 64 | self.v_proj = nn.Linear(embed_dim, embed_dim) |
| 65 | self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) |
| 66 | self.num_heads = num_heads |
| 67 | |
| 68 | def forward(self, x): |
| 69 | x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC |