(self,
input_feats,
num_frames=240,
latent_dim=512,
ff_size=1024,
num_layers=8,
num_heads=8,
dropout=0,
activation="gelu",
num_text_layers=4,
text_latent_dim=256,
text_ff_size=2048,
text_num_heads=4,
no_clip=False,
no_eff=False,
**kargs)
| 287 | |
| 288 | class MotionTransformer(nn.Module): |
| 289 | def __init__(self, |
| 290 | input_feats, |
| 291 | num_frames=240, |
| 292 | latent_dim=512, |
| 293 | ff_size=1024, |
| 294 | num_layers=8, |
| 295 | num_heads=8, |
| 296 | dropout=0, |
| 297 | activation="gelu", |
| 298 | num_text_layers=4, |
| 299 | text_latent_dim=256, |
| 300 | text_ff_size=2048, |
| 301 | text_num_heads=4, |
| 302 | no_clip=False, |
| 303 | no_eff=False, |
| 304 | **kargs): |
| 305 | super().__init__() |
| 306 | |
| 307 | self.num_frames = num_frames |
| 308 | self.latent_dim = latent_dim |
| 309 | self.ff_size = ff_size |
| 310 | self.num_layers = num_layers |
| 311 | self.num_heads = num_heads |
| 312 | self.dropout = dropout |
| 313 | self.activation = activation |
| 314 | self.input_feats = input_feats |
| 315 | self.time_embed_dim = latent_dim * 4 |
| 316 | self.sequence_embedding = nn.Parameter(torch.randn(num_frames, latent_dim)) |
| 317 | |
| 318 | # Text Transformer |
| 319 | self.clip, _ = clip.load('ViT-B/32', "cpu") |
| 320 | if no_clip: |
| 321 | self.clip.initialize_parameters() |
| 322 | else: |
| 323 | set_requires_grad(self.clip, False) |
| 324 | if text_latent_dim != 512: |
| 325 | self.text_pre_proj = nn.Linear(512, text_latent_dim) |
| 326 | else: |
| 327 | self.text_pre_proj = nn.Identity() |
| 328 | textTransEncoderLayer = nn.TransformerEncoderLayer( |
| 329 | d_model=text_latent_dim, |
| 330 | nhead=text_num_heads, |
| 331 | dim_feedforward=text_ff_size, |
| 332 | dropout=dropout, |
| 333 | activation=activation) |
| 334 | self.textTransEncoder = nn.TransformerEncoder( |
| 335 | textTransEncoderLayer, |
| 336 | num_layers=num_text_layers) |
| 337 | self.text_ln = nn.LayerNorm(text_latent_dim) |
| 338 | self.text_proj = nn.Sequential( |
| 339 | nn.Linear(text_latent_dim, self.time_embed_dim) |
| 340 | ) |
| 341 | |
| 342 | # Input Embedding |
| 343 | self.joint_embed = nn.Linear(self.input_feats, self.latent_dim) |
| 344 | |
| 345 | self.time_embed = nn.Sequential( |
| 346 | nn.Linear(self.latent_dim, self.time_embed_dim), |
nothing calls this directly
no test coverage detected