| 335 | } |
| 336 | |
| 337 | class WanAttentionBlock(nn.Module): |
| 338 | |
| 339 | def __init__(self, |
| 340 | cross_attn_type, |
| 341 | dim, |
| 342 | ffn_dim, |
| 343 | num_heads, |
| 344 | window_size=(-1, -1), |
| 345 | qk_norm=True, |
| 346 | cross_attn_norm=False, |
| 347 | eps=1e-6, |
| 348 | ): |
| 349 | super().__init__() |
| 350 | self.dim = dim |
| 351 | self.ffn_dim = ffn_dim |
| 352 | self.num_heads = num_heads |
| 353 | self.window_size = window_size |
| 354 | self.qk_norm = qk_norm |
| 355 | self.cross_attn_norm = cross_attn_norm |
| 356 | self.eps = eps |
| 357 | |
| 358 | # layers |
| 359 | self.norm1 = WanLayerNorm(dim, eps) |
| 360 | self.self_attn = WanSelfAttention(dim, num_heads, window_size, qk_norm, eps) |
| 361 | self.norm3 = WanLayerNorm( |
| 362 | dim, eps, |
| 363 | elementwise_affine=True) if cross_attn_norm else nn.Identity() |
| 364 | self.cross_attn = WANX_CROSSATTENTION_CLASSES[cross_attn_type]( |
| 365 | dim, num_heads, (-1, -1), qk_norm, eps) |
| 366 | self.norm2 = WanLayerNorm(dim, eps) |
| 367 | self.ffn = nn.Sequential( |
| 368 | nn.Linear(dim, ffn_dim), nn.GELU(approximate='tanh'), |
| 369 | nn.Linear(ffn_dim, dim)) |
| 370 | |
| 371 | # m2m cross attn |
| 372 | # self.ref_motion_cross_attn = nn.MultiheadAttention(dim, num_heads, dropout=0) |
| 373 | self.ref_motion_cross_attn = WanT2VCrossAttention(dim, num_heads, (-1, -1), qk_norm, eps) |
| 374 | self.norm4 = WanLayerNorm( |
| 375 | dim, eps, |
| 376 | elementwise_affine=True) if cross_attn_norm else nn.Identity() |
| 377 | |
| 378 | # modulation |
| 379 | self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) |
| 380 | |
| 381 | |
| 382 | def forward( |
| 383 | self, |
| 384 | x, |
| 385 | e, |
| 386 | seq_lens, |
| 387 | freqs, |
| 388 | context, |
| 389 | context_lens, |
| 390 | ref_motion, # [B, T, C] |
| 391 | ref_motion_lens, # [B] |
| 392 | attend_to_text_mask=None, # [B]: 1 for attend to text, 0 for attend to ref motion |
| 393 | ): |
| 394 | assert e.dtype == torch.float32 |