(
self,
embed_dims,
num_heads,
feedforward_channels,
drop_rate=0.0,
attn_drop_rate=0.0,
drop_path_rate=0.0,
qkv_bias=True,
act_cfg=dict(type="GELU"),
norm_cfg=dict(type="LN"),
batch_first=True,
sr_ratio=1,
with_cp=False,
)
| 233 | """ |
| 234 | |
| 235 | def __init__( |
| 236 | self, |
| 237 | embed_dims, |
| 238 | num_heads, |
| 239 | feedforward_channels, |
| 240 | drop_rate=0.0, |
| 241 | attn_drop_rate=0.0, |
| 242 | drop_path_rate=0.0, |
| 243 | qkv_bias=True, |
| 244 | act_cfg=dict(type="GELU"), |
| 245 | norm_cfg=dict(type="LN"), |
| 246 | batch_first=True, |
| 247 | sr_ratio=1, |
| 248 | with_cp=False, |
| 249 | ): |
| 250 | super(TransformerEncoderLayer, self).__init__() |
| 251 | |
| 252 | # The ret[0] of build_norm_layer is norm name. |
| 253 | self.norm1 = build_norm_layer(norm_cfg, embed_dims)[1] |
| 254 | |
| 255 | self.attn = EfficientMultiheadAttention( |
| 256 | embed_dims=embed_dims, |
| 257 | num_heads=num_heads, |
| 258 | attn_drop=attn_drop_rate, |
| 259 | proj_drop=drop_rate, |
| 260 | dropout_layer=dict(type="DropPath", drop_prob=drop_path_rate), |
| 261 | batch_first=batch_first, |
| 262 | qkv_bias=qkv_bias, |
| 263 | norm_cfg=norm_cfg, |
| 264 | sr_ratio=sr_ratio, |
| 265 | ) |
| 266 | |
| 267 | # The ret[0] of build_norm_layer is norm name. |
| 268 | self.norm2 = build_norm_layer(norm_cfg, embed_dims)[1] |
| 269 | |
| 270 | self.ffn = MixFFN( |
| 271 | embed_dims=embed_dims, |
| 272 | feedforward_channels=feedforward_channels, |
| 273 | ffn_drop=drop_rate, |
| 274 | dropout_layer=dict(type="DropPath", drop_prob=drop_path_rate), |
| 275 | act_cfg=act_cfg, |
| 276 | ) |
| 277 | |
| 278 | self.with_cp = with_cp |
| 279 | |
| 280 | def forward(self, x, hw_shape): |
| 281 | def _inner_forward(x): |
nothing calls this directly
no test coverage detected