Vision Transformer
| 260 | |
| 261 | |
| 262 | class VisionTransformer(nn.Module): |
| 263 | """ Vision Transformer """ |
| 264 | def __init__(self, img_size=[224], patch_size=16, in_chans=3, num_classes=0, embed_dim=768, depth=12, |
| 265 | num_heads=12, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop_rate=0., attn_drop_rate=0., |
| 266 | drop_path_rate=0., norm_layer=nn.LayerNorm, **kwargs): |
| 267 | super().__init__() |
| 268 | self.num_features = self.embed_dim = embed_dim |
| 269 | self.num_heads = num_heads |
| 270 | |
| 271 | self.patch_embed = PatchEmbed( |
| 272 | img_size=img_size[0], patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim) |
| 273 | num_patches = self.patch_embed.num_patches |
| 274 | |
| 275 | self.pos_embed = nn.Parameter(torch.zeros(1, num_patches, embed_dim)) |
| 276 | self.pos_drop = nn.Dropout(p=drop_rate) |
| 277 | |
| 278 | dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] # stochastic depth decay rule |
| 279 | self.blocks = nn.ModuleList([ |
| 280 | Block( |
| 281 | dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, |
| 282 | drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) |
| 283 | for i in range(depth)]) |
| 284 | |
| 285 | # Classifier head |
| 286 | self.head = nn.Linear(embed_dim, num_classes) if num_classes > 0 else nn.Identity() |
| 287 | |
| 288 | trunc_normal_(self.pos_embed, std=.02) |
| 289 | self.apply(self._init_weights) |
| 290 | |
| 291 | def _init_weights(self, m): |
| 292 | if isinstance(m, nn.Linear): |
| 293 | trunc_normal_(m.weight, std=.02) |
| 294 | if isinstance(m, nn.Linear) and m.bias is not None: |
| 295 | nn.init.constant_(m.bias, 0) |
| 296 | elif isinstance(m, nn.LayerNorm): |
| 297 | nn.init.constant_(m.bias, 0) |
| 298 | nn.init.constant_(m.weight, 1.0) |
| 299 | |
| 300 | def interpolate_pos_encoding(self, x, w, h): |
| 301 | npatch = x.shape[1] |
| 302 | N = self.pos_embed.shape[1] |
| 303 | if npatch == N and w == h: |
| 304 | return self.pos_embed |
| 305 | patch_pos_embed = self.pos_embed |
| 306 | dim = x.shape[-1] |
| 307 | w0 = w // self.patch_embed.patch_size |
| 308 | h0 = h // self.patch_embed.patch_size |
| 309 | # we add a small number to avoid floating point error in the interpolation |
| 310 | # see discussion at https://github.com/facebookresearch/dino/issues/8 |
| 311 | w0, h0 = w0 + 0.1, h0 + 0.1 |
| 312 | patch_pos_embed = nn.functional.interpolate( |
| 313 | patch_pos_embed.reshape(1, int(math.sqrt(N)), int(math.sqrt(N)), dim).permute(0, 3, 1, 2), |
| 314 | scale_factor=(w0 / math.sqrt(N), h0 / math.sqrt(N)), |
| 315 | mode='bicubic', |
| 316 | ) |
| 317 | assert int(w0) == patch_pos_embed.shape[-2] and int(h0) == patch_pos_embed.shape[-1] |
| 318 | return patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim) |
| 319 |