Implements one stage in Swin Transformer. Args: embed_dims (int): The feature dimension. num_heads (int): Parallel attention heads. feedforward_channels (int): The hidden dimension for FFNs. depth (int): The number of blocks in this stage. window_size (in
| 375 | |
| 376 | |
| 377 | class SwinBlockSequence(BaseModule): |
| 378 | """Implements one stage in Swin Transformer. |
| 379 | |
| 380 | Args: |
| 381 | embed_dims (int): The feature dimension. |
| 382 | num_heads (int): Parallel attention heads. |
| 383 | feedforward_channels (int): The hidden dimension for FFNs. |
| 384 | depth (int): The number of blocks in this stage. |
| 385 | window_size (int, optional): The local window scale. Default: 7. |
| 386 | qkv_bias (bool, optional): enable bias for qkv if True. Default: True. |
| 387 | qk_scale (float | None, optional): Override default qk scale of |
| 388 | head_dim ** -0.5 if set. Default: None. |
| 389 | drop_rate (float, optional): Dropout rate. Default: 0. |
| 390 | attn_drop_rate (float, optional): Attention dropout rate. Default: 0. |
| 391 | drop_path_rate (float | list[float], optional): Stochastic depth |
| 392 | rate. Default: 0. |
| 393 | downsample (BaseModule | None, optional): The downsample operation |
| 394 | module. Default: None. |
| 395 | act_cfg (dict, optional): The config dict of activation function. |
| 396 | Default: dict(type='GELU'). |
| 397 | norm_cfg (dict, optional): The config dict of normalization. |
| 398 | Default: dict(type='LN'). |
| 399 | with_cp (bool, optional): Use checkpoint or not. Using checkpoint |
| 400 | will save some memory while slowing down the training speed. |
| 401 | Default: False. |
| 402 | init_cfg (dict | list | None, optional): The init config. |
| 403 | Default: None. |
| 404 | """ |
| 405 | |
| 406 | def __init__(self, |
| 407 | embed_dims, |
| 408 | num_heads, |
| 409 | feedforward_channels, |
| 410 | depth, |
| 411 | window_size=7, |
| 412 | qkv_bias=True, |
| 413 | qk_scale=None, |
| 414 | drop_rate=0., |
| 415 | attn_drop_rate=0., |
| 416 | drop_path_rate=0., |
| 417 | downsample=None, |
| 418 | act_cfg=dict(type='GELU'), |
| 419 | norm_cfg=dict(type='LN'), |
| 420 | with_cp=False, |
| 421 | init_cfg=None): |
| 422 | super().__init__(init_cfg=init_cfg) |
| 423 | |
| 424 | if isinstance(drop_path_rate, list): |
| 425 | drop_path_rates = drop_path_rate |
| 426 | assert len(drop_path_rates) == depth |
| 427 | else: |
| 428 | drop_path_rates = [deepcopy(drop_path_rate) for _ in range(depth)] |
| 429 | |
| 430 | self.blocks = ModuleList() |
| 431 | for i in range(depth): |
| 432 | block = SwinBlock( |
| 433 | embed_dims=embed_dims, |
| 434 | num_heads=num_heads, |