| 382 | |
| 383 | |
| 384 | class WanAttentionBlock(nn.Module): |
| 385 | |
| 386 | def __init__(self, |
| 387 | cross_attn_type, |
| 388 | dim, |
| 389 | ffn_dim, |
| 390 | num_heads, |
| 391 | window_size=(-1, -1), |
| 392 | qk_norm=True, |
| 393 | cross_attn_norm=False, |
| 394 | eps=1e-6): |
| 395 | super().__init__() |
| 396 | self.dim = dim |
| 397 | self.ffn_dim = ffn_dim |
| 398 | self.num_heads = num_heads |
| 399 | self.window_size = window_size |
| 400 | self.qk_norm = qk_norm |
| 401 | self.cross_attn_norm = cross_attn_norm |
| 402 | self.eps = eps |
| 403 | |
| 404 | # layers |
| 405 | self.norm1 = WanLayerNorm(dim, eps) |
| 406 | self.self_attn = WanSelfAttention(dim, num_heads, window_size, qk_norm, |
| 407 | eps) |
| 408 | self.norm3 = WanLayerNorm( |
| 409 | dim, eps, |
| 410 | elementwise_affine=True) if cross_attn_norm else nn.Identity() |
| 411 | self.cross_attn = WANX_CROSSATTENTION_CLASSES[cross_attn_type]( |
| 412 | dim, num_heads, (-1, -1), qk_norm, eps) |
| 413 | self.norm2 = WanLayerNorm(dim, eps) |
| 414 | self.ffn = nn.Sequential( |
| 415 | nn.Linear(dim, ffn_dim), nn.GELU(approximate='tanh'), |
| 416 | nn.Linear(ffn_dim, dim)) |
| 417 | |
| 418 | # modulation |
| 419 | self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) |
| 420 | |
| 421 | def forward( |
| 422 | self, |
| 423 | x, |
| 424 | e, |
| 425 | seq_lens, |
| 426 | freqs, |
| 427 | context, |
| 428 | context_lens, |
| 429 | ): |
| 430 | assert e.dtype == torch.float32 |
| 431 | with amp.autocast(dtype=torch.float32, device_type="cuda"): |
| 432 | e = (self.modulation.to(dtype=e.dtype, device=e.device) + e).chunk(6, dim=1) |
| 433 | assert e[0].dtype == torch.float32 |
| 434 | |
| 435 | # self-attention |
| 436 | y = self.self_attn( |
| 437 | self.norm1(x).float() * (1 + e[1]) + e[0], seq_lens, |
| 438 | freqs) |
| 439 | with amp.autocast(dtype=torch.float32, device_type="cuda"): |
| 440 | x = x + y * e[2] |
| 441 | |