| 121 | return x |
| 122 | |
| 123 | class ApertureAttentionBlock(nn.Module): |
| 124 | |
| 125 | def __init__(self, embed_dim: int, num_heads: int, ffn_dim: int, drop_path=0., layerscale=False, |
| 126 | norm_layer=nn.LayerNorm, layer_init_values=1e-5): |
| 127 | super().__init__() |
| 128 | self.layerscale = layerscale |
| 129 | self.embed_dim = embed_dim |
| 130 | self.attention_layer_norm = norm_layer(self.embed_dim, eps=1e-6) |
| 131 | self.attention = ApertureAwareAttention(embed_dim, num_heads) |
| 132 | self.drop_path = DropPath(drop_path) |
| 133 | self.final_layer_norm = norm_layer(self.embed_dim, eps=1e-6) |
| 134 | self.ffn = FeedForwardNetwork(embed_dim, ffn_dim) |
| 135 | self.pos = DWConv2d(embed_dim, 3, 1, 1) |
| 136 | |
| 137 | if layerscale: |
| 138 | self.gamma_1 = nn.Parameter(layer_init_values * torch.ones(1, 1, 1, embed_dim), requires_grad=True) |
| 139 | self.gamma_2 = nn.Parameter(layer_init_values * torch.ones(1, 1, 1, embed_dim), requires_grad=True) |
| 140 | |
| 141 | def forward(self, x: torch.Tensor, attention_rel_pos=None): |
| 142 | x = x + self.pos(x) # InitiaL 3 X 3 dwconv |
| 143 | if self.layerscale: |
| 144 | x = x + self.drop_path( |
| 145 | self.gamma_1 * self.attention(self.attention_layer_norm(x), attention_rel_pos)) |
| 146 | x = x + self.drop_path(self.gamma_2 * self.ffn(self.final_layer_norm(x))) |
| 147 | else: |
| 148 | x = x + self.drop_path( |
| 149 | self.attention(self.attention_layer_norm(x), attention_rel_pos)) |
| 150 | x = x + self.drop_path(self.ffn(self.final_layer_norm(x))) |
| 151 | return x |
| 152 | |
| 153 | class BlockMod(nn.Module): |
| 154 | def __init__(self, channels: int, dw_expand: float = 1., ffn_expand: int = 2, drop_out_rate: float = 0., |