| 155 | return y |
| 156 | |
| 157 | class FFN(nn.Module): |
| 158 | |
| 159 | def __init__(self, latent_dim, ffn_dim, dropout, time_embed_dim): |
| 160 | super().__init__() |
| 161 | self.linear1 = nn.Linear(latent_dim, ffn_dim) |
| 162 | self.linear2 = zero_module(nn.Linear(ffn_dim, latent_dim)) |
| 163 | self.activation = nn.GELU() |
| 164 | self.dropout = nn.Dropout(dropout) |
| 165 | self.proj_out = StylizationBlock(latent_dim, time_embed_dim, dropout) |
| 166 | |
| 167 | def forward(self, x, emb): |
| 168 | y = self.linear2(self.dropout(self.activation(self.linear1(x)))) |
| 169 | y = x + self.proj_out(y, emb) |
| 170 | return y |
| 171 | |
| 172 | |
| 173 | class LinearTemporalDiffusionTransformerDecoderLayer(nn.Module): |