| 181 | |
| 182 | |
| 183 | class SFFN(nn.Module): |
| 184 | |
| 185 | def __init__(self, latent_dim, ffn_dim, dropout, time_embed_dim, **kwargs): |
| 186 | super().__init__() |
| 187 | self.linear1_list = nn.ModuleList() |
| 188 | self.linear2_list = nn.ModuleList() |
| 189 | for i in range(8): |
| 190 | self.linear1_list.append(nn.Linear(latent_dim, ffn_dim)) |
| 191 | self.linear2_list.append(nn.Linear(ffn_dim, latent_dim)) |
| 192 | self.activation = nn.GELU() |
| 193 | self.dropout = nn.Dropout(dropout) |
| 194 | self.proj_out = StylizationBlock(latent_dim * 8, time_embed_dim, |
| 195 | dropout) |
| 196 | |
| 197 | def forward(self, x, emb, **kwargs): |
| 198 | B, T, D = x.shape |
| 199 | x = x.reshape(B, T, 8, -1) |
| 200 | output = [] |
| 201 | for i in range(8): |
| 202 | feat = x[:, :, i].contiguous() |
| 203 | feat = self.dropout(self.activation(self.linear1_list[i](feat))) |
| 204 | feat = self.linear2_list[i](feat) |
| 205 | output.append(feat) |
| 206 | y = torch.cat(output, dim=-1) |
| 207 | y = x.reshape(B, T, D) + self.proj_out(y, emb) |
| 208 | return y |
| 209 | |
| 210 | |
| 211 | class DecoderLayer(nn.Module): |