Forward pass of ConditionalFlowMatchingTransformer. t: (B,) tensor of diffusion timesteps [0, 1] x: (B, L, 512) : tensor of sequence of motion latent wa: (B, L, 512) / tensor sequence of wa latent wp: (B, L, 6) / tensor sequence of wp latent wr: (B, 512) / tensor of refere
(self, t, x, wa, wr, we, prev_x = None, prev_wa = None, train = True, **kwargs)
| 266 | |
| 267 | |
| 268 | def forward(self, t, x, wa, wr, we, prev_x = None, prev_wa = None, train = True, **kwargs) -> torch.Tensor: |
| 269 | """ |
| 270 | Forward pass of ConditionalFlowMatchingTransformer. |
| 271 | |
| 272 | t: (B,) tensor of diffusion timesteps [0, 1] |
| 273 | x: (B, L, 512) : tensor of sequence of motion latent |
| 274 | |
| 275 | wa: (B, L, 512) / tensor sequence of wa latent |
| 276 | wp: (B, L, 6) / tensor sequence of wp latent |
| 277 | wr: (B, 512) / tensor of reference motion latent (i.e., r -> s) |
| 278 | we: (B, 1, 7) / tensor of emotion latent |
| 279 | |
| 280 | prev_x: (B, L', 512) / previous x for auto-regressive generation |
| 281 | prev_wa: (B, L', 512) / previous audio for auto-regressive generation |
| 282 | """ |
| 283 | |
| 284 | # time encoding |
| 285 | t = self.t_embedder(t).unsqueeze(1) # (N, D) |
| 286 | |
| 287 | # condition encoding |
| 288 | wa = self.sequence_embedder(wa, dropout_prob = self.opt.audio_dropout_prob, train=train) |
| 289 | wr = self.sequence_embedder(wr.unsqueeze(1), dropout_prob = self.opt.ref_dropout_prob, train=train) |
| 290 | we = self.sequence_embedder(we, dropout_prob = self.opt.emotion_dropout_prob, train=train) |
| 291 | |
| 292 | # previous condition encoding |
| 293 | if prev_x is not None: |
| 294 | prev_x = self.sequence_embedder(prev_x, dropout_prob=0.5, train=train) |
| 295 | prev_wa = self.sequence_embedder(prev_wa, dropout_prob=0.5, train=train) |
| 296 | |
| 297 | x = torch.cat([prev_x, x], dim=1) |
| 298 | wa = torch.cat([prev_wa, wa], dim=1) |
| 299 | |
| 300 | x = self.x_embedder(x) |
| 301 | x = x + self.pos_embed # (N, L + L', D), where T = opt.wav2vec_sec * opt.fps, D = dim_w |
| 302 | |
| 303 | wr = wr.repeat(1, wa.shape[1], 1) |
| 304 | we = we.repeat(1, wa.shape[1], 1) |
| 305 | |
| 306 | c = torch.cat([wr, wa, we], dim=-1) |
| 307 | c = self.c_embedder(c) |
| 308 | c = t + c |
| 309 | |
| 310 | # forwarding FMT Blocks |
| 311 | for block in self.blocks: |
| 312 | x = block(x, c, self.alignment_mask) # (N, T, D) |
| 313 | return self.decoder(x, c) |
| 314 | |
| 315 | @torch.no_grad() |
| 316 | def forward_with_cfv(self, t, x, wa, wr, we, prev_x, prev_wa, a_cfg_scale=1.0, r_cfg_scale=1.0, e_cfg_scale=1.0, **kwargs) -> torch.Tensor: |
no test coverage detected