| 153 | weight_init(self) |
| 154 | |
| 155 | class MSA_head(nn.Module): |
| 156 | def __init__(self, mode='dilation',dim=128, num_heads=8, ffn_expansion_factor=4, bias=False, LayerNorm_type='WithBias'): |
| 157 | super(MSA_head, self).__init__() |
| 158 | self.norm1 = LayerNorm(dim, LayerNorm_type) |
| 159 | self.attn = Attention(dim, num_heads, bias,mode) |
| 160 | self.norm2 = LayerNorm(dim, LayerNorm_type) |
| 161 | self.ffn = FeedForward(dim, ffn_expansion_factor, bias) |
| 162 | |
| 163 | def forward(self, x,mask=None): |
| 164 | x = x + self.attn(self.norm1(x),mask) |
| 165 | x = x + self.ffn(self.norm2(x)) |
| 166 | return x |
| 167 | |
| 168 | def initialize(self): |
| 169 | weight_init(self) |
| 170 | |
| 171 | class MSA_module(nn.Module): |
| 172 | def __init__(self, dim=128): |