Focal Modulation Block. Args: dim (int): Number of input channels. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. drop (float, optional): Dropout rate. Default: 0.0 drop_path (float, optional): Stochastic depth rate. Default: 0.0 act_layer
| 116 | return x_out |
| 117 | |
| 118 | class FocalModulationBlock(nn.Module): |
| 119 | """ Focal Modulation Block. |
| 120 | |
| 121 | Args: |
| 122 | dim (int): Number of input channels. |
| 123 | mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. |
| 124 | drop (float, optional): Dropout rate. Default: 0.0 |
| 125 | drop_path (float, optional): Stochastic depth rate. Default: 0.0 |
| 126 | act_layer (nn.Module, optional): Activation layer. Default: nn.GELU |
| 127 | norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm |
| 128 | focal_level (int): number of focal levels |
| 129 | focal_window (int): focal kernel size at level 1 |
| 130 | """ |
| 131 | |
| 132 | def __init__(self, dim, mlp_ratio=4., drop=0., drop_path=0., |
| 133 | act_layer=nn.GELU, norm_layer=nn.LayerNorm, |
| 134 | focal_level=2, focal_window=9, |
| 135 | use_postln=False, use_postln_in_modulation=False, |
| 136 | scaling_modulator=False, |
| 137 | use_layerscale=False, |
| 138 | layerscale_value=1e-4): |
| 139 | super().__init__() |
| 140 | self.dim = dim |
| 141 | self.mlp_ratio = mlp_ratio |
| 142 | self.focal_window = focal_window |
| 143 | self.focal_level = focal_level |
| 144 | self.use_postln = use_postln |
| 145 | self.use_layerscale = use_layerscale |
| 146 | |
| 147 | self.norm1 = norm_layer(dim) |
| 148 | self.modulation = FocalModulation( |
| 149 | dim, focal_window=self.focal_window, focal_level=self.focal_level, proj_drop=drop, use_postln_in_modulation=use_postln_in_modulation, scaling_modulator=scaling_modulator |
| 150 | ) |
| 151 | |
| 152 | self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() |
| 153 | self.norm2 = norm_layer(dim) |
| 154 | mlp_hidden_dim = int(dim * mlp_ratio) |
| 155 | self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) |
| 156 | |
| 157 | self.H = None |
| 158 | self.W = None |
| 159 | |
| 160 | self.gamma_1 = 1.0 |
| 161 | self.gamma_2 = 1.0 |
| 162 | if self.use_layerscale: |
| 163 | self.gamma_1 = nn.Parameter(layerscale_value * torch.ones((dim)), requires_grad=True) |
| 164 | self.gamma_2 = nn.Parameter(layerscale_value * torch.ones((dim)), requires_grad=True) |
| 165 | |
| 166 | def forward(self, x): |
| 167 | """ Forward function. |
| 168 | |
| 169 | Args: |
| 170 | x: Input feature, tensor size (B, H*W, C). |
| 171 | H, W: Spatial resolution of the input feature. |
| 172 | """ |
| 173 | B, L, C = x.shape |
| 174 | H, W = self.H, self.W |
| 175 | assert L == H * W, "input feature has wrong size" |