| 154 | |
| 155 | |
| 156 | class AttentionPool(nn.Module): |
| 157 | |
| 158 | def __init__(self, |
| 159 | dim, |
| 160 | mlp_ratio, |
| 161 | num_heads, |
| 162 | activation='gelu', |
| 163 | proj_dropout=0.0, |
| 164 | norm_eps=1e-5): |
| 165 | assert dim % num_heads == 0 |
| 166 | super().__init__() |
| 167 | self.dim = dim |
| 168 | self.mlp_ratio = mlp_ratio |
| 169 | self.num_heads = num_heads |
| 170 | self.head_dim = dim // num_heads |
| 171 | self.proj_dropout = proj_dropout |
| 172 | self.norm_eps = norm_eps |
| 173 | |
| 174 | # layers |
| 175 | gain = 1.0 / math.sqrt(dim) |
| 176 | self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) |
| 177 | self.to_q = nn.Linear(dim, dim) |
| 178 | self.to_kv = nn.Linear(dim, dim * 2) |
| 179 | self.proj = nn.Linear(dim, dim) |
| 180 | self.norm = LayerNorm(dim, eps=norm_eps) |
| 181 | self.mlp = nn.Sequential( |
| 182 | nn.Linear(dim, int(dim * mlp_ratio)), |
| 183 | QuickGELU() if activation == 'quick_gelu' else nn.GELU(), |
| 184 | nn.Linear(int(dim * mlp_ratio), dim), nn.Dropout(proj_dropout)) |
| 185 | |
| 186 | def forward(self, x): |
| 187 | """ |
| 188 | x: [B, L, C]. |
| 189 | """ |
| 190 | b, s, c, n, d = *x.size(), self.num_heads, self.head_dim |
| 191 | |
| 192 | # compute query, key, value |
| 193 | q = self.to_q(self.cls_embedding).view(1, 1, n, d).expand(b, -1, -1, -1) |
| 194 | k, v = self.to_kv(x).view(b, s, 2, n, d).unbind(2) |
| 195 | |
| 196 | # compute attention |
| 197 | x = flash_attention(q, k, v, version=2) |
| 198 | x = x.reshape(b, 1, c) |
| 199 | |
| 200 | # output |
| 201 | x = self.proj(x) |
| 202 | x = F.dropout(x, self.proj_dropout, self.training) |
| 203 | |
| 204 | # mlp |
| 205 | x = x + self.mlp(self.norm(x)) |
| 206 | return x[:, 0] |
| 207 | |
| 208 | |
| 209 | class VisionTransformer(nn.Module): |