| 42 | """ |
| 43 | |
| 44 | def __init__(self, |
| 45 | embed_dims, |
| 46 | num_heads, |
| 47 | feedforward_channels, |
| 48 | drop_rate=0., |
| 49 | attn_drop_rate=0., |
| 50 | drop_path_rate=0., |
| 51 | num_fcs=2, |
| 52 | qkv_bias=True, |
| 53 | act_cfg=dict(type='GELU'), |
| 54 | norm_cfg=dict(type='LN'), |
| 55 | batch_first=True): |
| 56 | super(TransformerEncoderLayer, self).__init__() |
| 57 | |
| 58 | self.norm1_name, norm1 = build_norm_layer( |
| 59 | norm_cfg, embed_dims, postfix=1) |
| 60 | self.add_module(self.norm1_name, norm1) |
| 61 | |
| 62 | self.attn = MultiheadAttention( |
| 63 | embed_dims=embed_dims, |
| 64 | num_heads=num_heads, |
| 65 | attn_drop=attn_drop_rate, |
| 66 | proj_drop=drop_rate, |
| 67 | dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate), |
| 68 | batch_first=batch_first, |
| 69 | bias=qkv_bias) |
| 70 | |
| 71 | self.norm2_name, norm2 = build_norm_layer( |
| 72 | norm_cfg, embed_dims, postfix=2) |
| 73 | self.add_module(self.norm2_name, norm2) |
| 74 | |
| 75 | self.ffn = FFN( |
| 76 | embed_dims=embed_dims, |
| 77 | feedforward_channels=feedforward_channels, |
| 78 | num_fcs=num_fcs, |
| 79 | ffn_drop=drop_rate, |
| 80 | dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate), |
| 81 | act_cfg=act_cfg) |
| 82 | |
| 83 | @property |
| 84 | def norm1(self): |