| 87 | |
| 88 | |
| 89 | class WanLayerNorm(nn.LayerNorm): |
| 90 | |
| 91 | def __init__(self, dim, eps=1e-6, elementwise_affine=False): |
| 92 | super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) |
| 93 | |
| 94 | def forward(self, x): |
| 95 | r""" |
| 96 | Args: |
| 97 | x(Tensor): Shape [B, L, C] |
| 98 | """ |
| 99 | return super().forward(x.float()).type_as(x) |
| 100 | |
| 101 | |
| 102 | class WanSelfAttention(nn.Module): |