| 90 | |
| 91 | |
| 92 | class WanLayerNorm(nn.LayerNorm): |
| 93 | |
| 94 | def __init__(self, dim, eps=1e-6, elementwise_affine=False): |
| 95 | super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) |
| 96 | |
| 97 | def forward(self, x): |
| 98 | r""" |
| 99 | Args: |
| 100 | x(Tensor): Shape [B, L, C] |
| 101 | """ |
| 102 | return super().forward(x.float()).type_as(x) |
| 103 | |
| 104 | |
| 105 | class WanSelfAttention(nn.Module): |