| 38 | """ |
| 39 | |
| 40 | def __init__( |
| 41 | self, |
| 42 | embedding_dim: int, |
| 43 | num_embeddings: int | None = None, |
| 44 | output_dim: int | None = None, |
| 45 | norm_elementwise_affine: bool = False, |
| 46 | norm_eps: float = 1e-5, |
| 47 | chunk_dim: int = 0, |
| 48 | ): |
| 49 | super().__init__() |
| 50 | |
| 51 | self.chunk_dim = chunk_dim |
| 52 | output_dim = output_dim or embedding_dim * 2 |
| 53 | |
| 54 | if num_embeddings is not None: |
| 55 | self.emb = nn.Embedding(num_embeddings, embedding_dim) |
| 56 | else: |
| 57 | self.emb = None |
| 58 | |
| 59 | self.silu = nn.SiLU() |
| 60 | self.linear = nn.Linear(embedding_dim, output_dim) |
| 61 | self.norm = nn.LayerNorm(output_dim // 2, norm_eps, norm_elementwise_affine) |
| 62 | |
| 63 | def forward( |
| 64 | self, x: torch.Tensor, timestep: torch.Tensor | None = None, temb: torch.Tensor | None = None |