| 125 | |
| 126 | |
| 127 | class DecoderLayer(nn.Module): |
| 128 | |
| 129 | def __init__( |
| 130 | self, |
| 131 | dim, |
| 132 | num_heads, |
| 133 | mlp_ratio=4.0, |
| 134 | qkv_bias=False, |
| 135 | qk_scale=None, |
| 136 | drop=0.0, |
| 137 | attn_drop=0.0, |
| 138 | drop_path=0.0, |
| 139 | act_layer=nn.GELU, |
| 140 | norm_layer=nn.LayerNorm, |
| 141 | epsilon=1e-6, |
| 142 | ): |
| 143 | super().__init__() |
| 144 | self.norm1 = norm_layer(dim, eps=epsilon) |
| 145 | self.mixer = Attention( |
| 146 | dim, |
| 147 | num_heads=num_heads, |
| 148 | qkv_bias=qkv_bias, |
| 149 | qk_scale=qk_scale, |
| 150 | attn_drop=attn_drop, |
| 151 | proj_drop=drop, |
| 152 | ) |
| 153 | |
| 154 | # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here |
| 155 | self.drop_path = DropPath(drop_path) if drop_path > 0.0 else Identity() |
| 156 | self.norm2 = norm_layer(dim, eps=epsilon) |
| 157 | mlp_hidden_dim = int(dim * mlp_ratio) |
| 158 | self.mlp_ratio = mlp_ratio |
| 159 | self.mlp = Mlp( |
| 160 | in_features=dim, |
| 161 | hidden_features=mlp_hidden_dim, |
| 162 | act_layer=act_layer, |
| 163 | drop=drop, |
| 164 | ) |
| 165 | |
| 166 | def forward(self, q, kv, key_mask=None): |
| 167 | x1 = self.norm1(q + self.drop_path(self.mixer(q, kv, key_mask))) |
| 168 | x = self.norm2(x1 + self.drop_path(self.mlp(x1))) |
| 169 | return x |
| 170 | |
| 171 | |
| 172 | class CPPDDecoder(nn.Module): |