| 271 | |
| 272 | |
| 273 | class DecoderBlock(nn.Module): |
| 274 | |
| 275 | def __init__( |
| 276 | self, |
| 277 | dim, |
| 278 | num_heads, |
| 279 | mlp_ratio=4.0, |
| 280 | qkv_bias=False, |
| 281 | drop=0.0, |
| 282 | attn_drop=0.0, |
| 283 | drop_path=0.0, |
| 284 | act_layer=nn.GELU, |
| 285 | norm_layer=nn.LayerNorm, |
| 286 | norm_mem=True, |
| 287 | rope=None, |
| 288 | ): |
| 289 | super().__init__() |
| 290 | self.norm1 = norm_layer(dim) |
| 291 | self.attn = Attention( |
| 292 | dim, |
| 293 | rope=rope, |
| 294 | num_heads=num_heads, |
| 295 | qkv_bias=qkv_bias, |
| 296 | attn_drop=attn_drop, |
| 297 | proj_drop=drop, |
| 298 | ) |
| 299 | self.cross_attn = CrossAttention( |
| 300 | dim, |
| 301 | rope=rope, |
| 302 | num_heads=num_heads, |
| 303 | qkv_bias=qkv_bias, |
| 304 | attn_drop=attn_drop, |
| 305 | proj_drop=drop, |
| 306 | ) |
| 307 | self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() |
| 308 | self.norm2 = norm_layer(dim) |
| 309 | self.norm3 = norm_layer(dim) |
| 310 | mlp_hidden_dim = int(dim * mlp_ratio) |
| 311 | self.mlp = Mlp( |
| 312 | in_features=dim, |
| 313 | hidden_features=mlp_hidden_dim, |
| 314 | act_layer=act_layer, |
| 315 | drop=drop, |
| 316 | ) |
| 317 | self.norm_y = norm_layer(dim) if norm_mem else nn.Identity() |
| 318 | |
| 319 | def forward(self, x, y, xpos, ypos, return_attn=False): |
| 320 | if return_attn: |
| 321 | self_attn_output, self_attn = self.attn(self.norm1(x), xpos, return_attn=True) |
| 322 | x = x + self.drop_path(self_attn_output) |
| 323 | y_ = self.norm_y(y) |
| 324 | cross_attn_output, cross_attn = self.cross_attn(self.norm2(x), y_, y_, xpos, ypos, return_attn=True) |
| 325 | x = x + self.drop_path(cross_attn_output) |
| 326 | x = x + self.drop_path(self.mlp(self.norm3(x))) |
| 327 | return x, y, self_attn, cross_attn |
| 328 | else: |
| 329 | x = x + self.drop_path(self.attn(self.norm1(x), xpos)) |
| 330 | y_ = self.norm_y(y) |
no outgoing calls
no test coverage detected