(self, dim, mlp_ratio=4., drop=0., drop_path=0.,
act_layer=nn.GELU, norm_layer=nn.LayerNorm,
focal_level=2, focal_window=9,
use_postln=False, use_postln_in_modulation=False,
scaling_modulator=False,
use_layerscale=False,
layerscale_value=1e-4)
| 130 | """ |
| 131 | |
| 132 | def __init__(self, dim, mlp_ratio=4., drop=0., drop_path=0., |
| 133 | act_layer=nn.GELU, norm_layer=nn.LayerNorm, |
| 134 | focal_level=2, focal_window=9, |
| 135 | use_postln=False, use_postln_in_modulation=False, |
| 136 | scaling_modulator=False, |
| 137 | use_layerscale=False, |
| 138 | layerscale_value=1e-4): |
| 139 | super().__init__() |
| 140 | self.dim = dim |
| 141 | self.mlp_ratio = mlp_ratio |
| 142 | self.focal_window = focal_window |
| 143 | self.focal_level = focal_level |
| 144 | self.use_postln = use_postln |
| 145 | self.use_layerscale = use_layerscale |
| 146 | |
| 147 | self.norm1 = norm_layer(dim) |
| 148 | self.modulation = FocalModulation( |
| 149 | dim, focal_window=self.focal_window, focal_level=self.focal_level, proj_drop=drop, use_postln_in_modulation=use_postln_in_modulation, scaling_modulator=scaling_modulator |
| 150 | ) |
| 151 | |
| 152 | self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() |
| 153 | self.norm2 = norm_layer(dim) |
| 154 | mlp_hidden_dim = int(dim * mlp_ratio) |
| 155 | self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) |
| 156 | |
| 157 | self.H = None |
| 158 | self.W = None |
| 159 | |
| 160 | self.gamma_1 = 1.0 |
| 161 | self.gamma_2 = 1.0 |
| 162 | if self.use_layerscale: |
| 163 | self.gamma_1 = nn.Parameter(layerscale_value * torch.ones((dim)), requires_grad=True) |
| 164 | self.gamma_2 = nn.Parameter(layerscale_value * torch.ones((dim)), requires_grad=True) |
| 165 | |
| 166 | def forward(self, x): |
| 167 | """ Forward function. |
nothing calls this directly
no test coverage detected