| 333 | class AttentionPool(nn.Module): |
| 334 | |
| 335 | def __init__(self, |
| 336 | dim, |
| 337 | mlp_ratio, |
| 338 | num_heads, |
| 339 | activation='gelu', |
| 340 | proj_dropout=0.0, |
| 341 | norm_eps=1e-5): |
| 342 | assert dim % num_heads == 0 |
| 343 | super().__init__() |
| 344 | self.dim = dim |
| 345 | self.mlp_ratio = mlp_ratio |
| 346 | self.num_heads = num_heads |
| 347 | self.head_dim = dim // num_heads |
| 348 | self.proj_dropout = proj_dropout |
| 349 | self.norm_eps = norm_eps |
| 350 | |
| 351 | # layers |
| 352 | gain = 1.0 / math.sqrt(dim) |
| 353 | self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) |
| 354 | self.to_q = nn.Linear(dim, dim) |
| 355 | self.to_kv = nn.Linear(dim, dim * 2) |
| 356 | self.proj = nn.Linear(dim, dim) |
| 357 | self.norm = LayerNorm(dim, eps=norm_eps) |
| 358 | self.mlp = nn.Sequential( |
| 359 | nn.Linear(dim, int(dim * mlp_ratio)), |
| 360 | QuickGELU() if activation == 'quick_gelu' else nn.GELU(), |
| 361 | nn.Linear(int(dim * mlp_ratio), dim), nn.Dropout(proj_dropout)) |
| 362 | |
| 363 | def forward(self, x): |
| 364 | """ |