Implements one encoder layer in Twins-SVT. Args: embed_dims (int): The feature dimension. num_heads (int): Parallel attention heads. feedforward_channels (int): The hidden dimension for FFNs. drop_rate (float): Probability of an element to be zeroed a
| 247 | |
| 248 | |
| 249 | class LSAEncoderLayer(BaseModule): |
| 250 | """Implements one encoder layer in Twins-SVT. |
| 251 | |
| 252 | Args: |
| 253 | embed_dims (int): The feature dimension. |
| 254 | num_heads (int): Parallel attention heads. |
| 255 | feedforward_channels (int): The hidden dimension for FFNs. |
| 256 | drop_rate (float): Probability of an element to be zeroed |
| 257 | after the feed forward layer. Default: 0.0. |
| 258 | attn_drop_rate (float, optional): Dropout ratio of attention weight. |
| 259 | Default: 0.0 |
| 260 | drop_path_rate (float): Stochastic depth rate. Default 0.0. |
| 261 | num_fcs (int): The number of fully-connected layers for FFNs. |
| 262 | Default: 2. |
| 263 | qkv_bias (bool): Enable bias for qkv if True. Default: True |
| 264 | qk_scale (float | None, optional): Override default qk scale of |
| 265 | head_dim ** -0.5 if set. Default: None. |
| 266 | act_cfg (dict): The activation config for FFNs. |
| 267 | Default: dict(type='GELU'). |
| 268 | norm_cfg (dict): Config dict for normalization layer. |
| 269 | Default: dict(type='LN'). |
| 270 | window_size (int): Window size of LSA. Default: 1. |
| 271 | init_cfg (dict, optional): The Config for initialization. |
| 272 | Defaults to None. |
| 273 | """ |
| 274 | |
| 275 | def __init__( |
| 276 | self, |
| 277 | embed_dims, |
| 278 | num_heads, |
| 279 | feedforward_channels, |
| 280 | drop_rate=0.0, |
| 281 | attn_drop_rate=0.0, |
| 282 | drop_path_rate=0.0, |
| 283 | num_fcs=2, |
| 284 | qkv_bias=True, |
| 285 | qk_scale=None, |
| 286 | act_cfg=dict(type="GELU"), |
| 287 | norm_cfg=dict(type="LN"), |
| 288 | window_size=1, |
| 289 | init_cfg=None, |
| 290 | ): |
| 291 | super(LSAEncoderLayer, self).__init__(init_cfg=init_cfg) |
| 292 | |
| 293 | self.norm1 = build_norm_layer(norm_cfg, embed_dims, postfix=1)[1] |
| 294 | self.attn = LocallyGroupedSelfAttention( |
| 295 | embed_dims, num_heads, qkv_bias, qk_scale, attn_drop_rate, drop_rate, window_size |
| 296 | ) |
| 297 | |
| 298 | self.norm2 = build_norm_layer(norm_cfg, embed_dims, postfix=2)[1] |
| 299 | self.ffn = FFN( |
| 300 | embed_dims=embed_dims, |
| 301 | feedforward_channels=feedforward_channels, |
| 302 | num_fcs=num_fcs, |
| 303 | ffn_drop=drop_rate, |
| 304 | dropout_layer=dict(type="DropPath", drop_prob=drop_path_rate), |
| 305 | act_cfg=act_cfg, |
| 306 | add_identity=False, |