| 35 | |
| 36 | |
| 37 | class MotionEncoder(nn.Module): |
| 38 | |
| 39 | def __init__(self, input_dim, latent_dim, ff_size, num_layers, num_heads, |
| 40 | dropout, activation): |
| 41 | super().__init__() |
| 42 | |
| 43 | self.input_feats = input_dim |
| 44 | self.latent_dim = latent_dim |
| 45 | self.ff_size = ff_size |
| 46 | self.num_layers = num_layers |
| 47 | self.num_heads = num_heads |
| 48 | self.dropout = dropout |
| 49 | self.activation = activation |
| 50 | |
| 51 | self.query_token = nn.Parameter(torch.randn(1, self.latent_dim)) |
| 52 | |
| 53 | self.embed_motion = nn.Linear(self.input_feats * 2, self.latent_dim) |
| 54 | self.sequence_pos_encoder = PositionalEncoding(self.latent_dim, |
| 55 | self.dropout, |
| 56 | max_len=2000) |
| 57 | |
| 58 | seqTransEncoderLayer = nn.TransformerEncoderLayer( |
| 59 | d_model=self.latent_dim, |
| 60 | nhead=self.num_heads, |
| 61 | dim_feedforward=self.ff_size, |
| 62 | dropout=self.dropout, |
| 63 | activation=self.activation) |
| 64 | self.transformer = nn.TransformerEncoder(seqTransEncoderLayer, |
| 65 | num_layers=self.num_layers) |
| 66 | self.out_ln = nn.LayerNorm(self.latent_dim) |
| 67 | self.out = nn.Linear(self.latent_dim, 512) |
| 68 | |
| 69 | def forward(self, motion, motion_mask): |
| 70 | x, mask = motion, motion_mask |
| 71 | B, T = x.shape[:2] |
| 72 | |
| 73 | x = x.reshape(B, T, 2, -1)[..., :-4].reshape(B, T, -1) |
| 74 | |
| 75 | x_emb = self.embed_motion(x) |
| 76 | |
| 77 | idx = torch.zeros(B, dtype=torch.long, device=x.device) |
| 78 | emb = torch.cat([self.query_token[idx][:, None], x_emb], dim=1) |
| 79 | |
| 80 | seq_mask = (mask > 0.5) |
| 81 | token_mask = torch.ones((B, 1), dtype=bool, device=x.device) |
| 82 | valid_mask = torch.cat([token_mask, seq_mask], dim=1) |
| 83 | |
| 84 | h = self.sequence_pos_encoder(emb) |
| 85 | |
| 86 | h = h.permute(1, 0, 2) |
| 87 | h = self.transformer(h, src_key_padding_mask=~valid_mask).permute( |
| 88 | 1, 0, 2) |
| 89 | h = self.out_ln(h) |
| 90 | motion_emb = self.out(h[:, 0]) |
| 91 | |
| 92 | return motion_emb |
| 93 | |
| 94 | |