| 10 | |
| 11 | |
| 12 | class FFN(nn.Module): |
| 13 | |
| 14 | def __init__(self, latent_dim, ffn_dim, dropout, time_embed_dim): |
| 15 | super().__init__() |
| 16 | self.latent_dim = latent_dim |
| 17 | self.linear1 = nn.Linear(latent_dim, ffn_dim) |
| 18 | self.linear2 = zero_module(nn.Linear(ffn_dim, latent_dim)) |
| 19 | self.activation = nn.GELU() |
| 20 | self.dropout = nn.Dropout(dropout) |
| 21 | self.proj_out = StylizationBlock(latent_dim, time_embed_dim, dropout) |
| 22 | |
| 23 | def forward(self, x, emb, **kwargs): |
| 24 | x1 = x[:, :, :self.latent_dim].contiguous() |
| 25 | x2 = x[:, :, self.latent_dim:].contiguous() |
| 26 | y1 = self.linear2(self.dropout(self.activation(self.linear1(x1)))) |
| 27 | y1 = x1 + self.proj_out(y1, emb) |
| 28 | y2 = self.linear2(self.dropout(self.activation(self.linear1(x2)))) |
| 29 | y2 = x2 + self.proj_out(y2, emb) |
| 30 | y = torch.cat((y1, y2), dim=-1) |
| 31 | return y |
| 32 | |
| 33 | |
| 34 | class DecoderLayer(nn.Module): |