| 73 | return x |
| 74 | |
| 75 | class TransformerBlock(nn.Module): |
| 76 | def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, drop=0., attn_drop=0., |
| 77 | drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): |
| 78 | super().__init__() |
| 79 | self.norm1 = norm_layer(dim) |
| 80 | |
| 81 | self.attn = Attention(dim, num_heads=num_heads, qkv_bias=qkv_bias, attn_drop=attn_drop, proj_drop=drop) |
| 82 | |
| 83 | # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here |
| 84 | self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() |
| 85 | self.norm2 = norm_layer(dim) |
| 86 | mlp_hidden_dim = int(dim * mlp_ratio) |
| 87 | self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) |
| 88 | |
| 89 | def forward(self, x, H, W): |
| 90 | x = x + self.drop_path(self.attn(self.norm1(x), H, W)) |
| 91 | x = x + self.drop_path(self.mlp(self.norm2(x))) |
| 92 | return x |
| 93 | |
| 94 | |
| 95 | class LadderSideAttentionFPN(nn.Module): |
no outgoing calls
no test coverage detected