(
self,
# backbone
input_resolution, patch_size, width, layers, heads, output_dim, backbone_drop_path_rate=0.,
use_checkpoint=False, checkpoint_num=[0], t_size=8, kernel_size=3, dw_reduction=1.5,
temporal_downsample=True,
no_lmhra=-False, double_lmhra=True,
# global block
return_list=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
n_layers=12, n_dim=768, n_head=12, mlp_factor=4.0, drop_path_rate=0.,
mlp_dropout=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5],
cls_dropout=0.5, num_classes=400,
frozen=False
)
| 276 | |
| 277 | class VisionTransformer(nn.Module): |
| 278 | def __init__( |
| 279 | self, |
| 280 | # backbone |
| 281 | input_resolution, patch_size, width, layers, heads, output_dim, backbone_drop_path_rate=0., |
| 282 | use_checkpoint=False, checkpoint_num=[0], t_size=8, kernel_size=3, dw_reduction=1.5, |
| 283 | temporal_downsample=True, |
| 284 | no_lmhra=-False, double_lmhra=True, |
| 285 | # global block |
| 286 | return_list=[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], |
| 287 | n_layers=12, n_dim=768, n_head=12, mlp_factor=4.0, drop_path_rate=0., |
| 288 | mlp_dropout=[0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5], |
| 289 | cls_dropout=0.5, num_classes=400, |
| 290 | frozen=False |
| 291 | ): |
| 292 | super().__init__() |
| 293 | self.input_resolution = input_resolution |
| 294 | self.output_dim = output_dim |
| 295 | padding = (kernel_size - 1) // 2 |
| 296 | if temporal_downsample: |
| 297 | self.conv1 = nn.Conv3d(3, width, (kernel_size, patch_size, patch_size), (2, patch_size, patch_size), (padding, 0, 0), bias=False) |
| 298 | t_size = t_size // 2 |
| 299 | else: |
| 300 | self.conv1 = nn.Conv3d(3, width, (1, patch_size, patch_size), (1, patch_size, patch_size), (0, 0, 0), bias=False) |
| 301 | |
| 302 | scale = width ** -0.5 |
| 303 | self.class_embedding = nn.Parameter(scale * torch.randn(width)) |
| 304 | self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)) |
| 305 | self.ln_pre = LayerNorm(width) |
| 306 | |
| 307 | |
| 308 | self.transformer = Transformer( |
| 309 | width, layers, heads, dw_reduction=dw_reduction, |
| 310 | backbone_drop_path_rate=backbone_drop_path_rate, |
| 311 | use_checkpoint=use_checkpoint, checkpoint_num=checkpoint_num, t_size=t_size, |
| 312 | no_lmhra=no_lmhra, double_lmhra=double_lmhra, |
| 313 | return_list=return_list, n_layers=n_layers, n_dim=n_dim, n_head=n_head, |
| 314 | mlp_factor=mlp_factor, drop_path_rate=drop_path_rate, mlp_dropout=mlp_dropout, |
| 315 | cls_dropout=cls_dropout, num_classes=num_classes, |
| 316 | frozen=frozen, |
| 317 | ) |
| 318 | |
| 319 | def forward(self, x): |
| 320 | x = self.conv1(x) # shape = [*, width, grid, grid] |
nothing calls this directly
no test coverage detected