r""" Norm layer modified to incorporate timestep embeddings. Parameters: embedding_dim (`int`): The size of each embedding vector. num_embeddings (`int`, *optional*): The size of the embeddings dictionary. output_dim (`int`, *optional*): norm_elementwise_affi
| 29 | |
| 30 | |
| 31 | class AdaLayerNorm(nn.Module): |
| 32 | r""" |
| 33 | Norm layer modified to incorporate timestep embeddings. |
| 34 | |
| 35 | Parameters: |
| 36 | embedding_dim (`int`): The size of each embedding vector. |
| 37 | num_embeddings (`int`, *optional*): The size of the embeddings dictionary. |
| 38 | output_dim (`int`, *optional*): |
| 39 | norm_elementwise_affine (`bool`, defaults to `False): |
| 40 | norm_eps (`bool`, defaults to `False`): |
| 41 | chunk_dim (`int`, defaults to `0`): |
| 42 | """ |
| 43 | |
| 44 | def __init__( |
| 45 | self, |
| 46 | embedding_dim: int, |
| 47 | num_embeddings: Optional[int] = None, |
| 48 | output_dim: Optional[int] = None, |
| 49 | norm_elementwise_affine: bool = False, |
| 50 | norm_eps: float = 1e-5, |
| 51 | chunk_dim: int = 0, |
| 52 | ): |
| 53 | super().__init__() |
| 54 | |
| 55 | self.chunk_dim = chunk_dim |
| 56 | output_dim = output_dim or embedding_dim * 2 |
| 57 | |
| 58 | if num_embeddings is not None: |
| 59 | self.emb = nn.Embedding(num_embeddings, embedding_dim) |
| 60 | else: |
| 61 | self.emb = None |
| 62 | |
| 63 | self.silu = nn.SiLU() |
| 64 | self.linear = nn.Linear(embedding_dim, output_dim) |
| 65 | self.norm = nn.LayerNorm(output_dim // 2, norm_eps, norm_elementwise_affine) |
| 66 | |
| 67 | def forward( |
| 68 | self, x: torch.Tensor, timestep: Optional[torch.Tensor] = None, temb: Optional[torch.Tensor] = None |
| 69 | ) -> torch.Tensor: |
| 70 | if self.emb is not None: |
| 71 | temb = self.emb(timestep) |
| 72 | |
| 73 | temb = self.linear(self.silu(temb)) |
| 74 | |
| 75 | if self.chunk_dim == 1: |
| 76 | # This is a bit weird why we have the order of "shift, scale" here and "scale, shift" in the |
| 77 | # other if-branch. This branch is specific to CogVideoX for now. |
| 78 | shift, scale = temb.chunk(2, dim=1) |
| 79 | shift = shift[:, None, :] |
| 80 | scale = scale[:, None, :] |
| 81 | else: |
| 82 | scale, shift = temb.chunk(2, dim=0) |
| 83 | |
| 84 | x = self.norm(x) * (1 + scale) + shift |
| 85 | return x |
| 86 | |
| 87 | |
| 88 | class FP32LayerNorm(nn.LayerNorm): |