| 94 | |
| 95 | @SUBMODULES.register_module() |
| 96 | class InterCLIP(BaseModule): |
| 97 | |
| 98 | def __init__(self, |
| 99 | input_dim=258, |
| 100 | latent_dim=1024, |
| 101 | ff_size=2048, |
| 102 | num_layers=8, |
| 103 | num_heads=8, |
| 104 | dropout=0.1, |
| 105 | activation="gelu", |
| 106 | init_cfg=None): |
| 107 | super().__init__() |
| 108 | self.latent_dim = latent_dim |
| 109 | self.motion_encoder = MotionEncoder(input_dim=input_dim, |
| 110 | latent_dim=latent_dim, |
| 111 | ff_size=ff_size, |
| 112 | num_layers=num_layers, |
| 113 | num_heads=num_heads, |
| 114 | dropout=dropout, |
| 115 | activation=activation) |
| 116 | |
| 117 | self.latent_dim = self.latent_dim |
| 118 | |
| 119 | clip_model, _ = clip.load("ViT-L/14@336px", device="cpu", jit=False) |
| 120 | |
| 121 | self.token_embedding = clip_model.token_embedding |
| 122 | self.positional_embedding = clip_model.positional_embedding |
| 123 | self.dtype = clip_model.dtype |
| 124 | self.latent_scale = nn.Parameter(torch.Tensor([1])) |
| 125 | |
| 126 | set_requires_grad(self.token_embedding, False) |
| 127 | |
| 128 | textTransEncoderLayer = nn.TransformerEncoderLayer( |
| 129 | d_model=768, |
| 130 | nhead=8, |
| 131 | dim_feedforward=ff_size, |
| 132 | dropout=0.1, |
| 133 | activation="gelu") |
| 134 | self.textTransEncoder = nn.TransformerEncoder(textTransEncoderLayer, |
| 135 | num_layers=8) |
| 136 | self.text_ln = nn.LayerNorm(768) |
| 137 | self.out = nn.Linear(768, 512) |
| 138 | |
| 139 | self.clip_training = "text_" |
| 140 | self.l1_criterion = torch.nn.L1Loss(reduction='mean') |
| 141 | assert init_cfg['type'] == 'Pretrained' |
| 142 | self.load_pretrained(init_cfg['checkpoint']) |
| 143 | |
| 144 | def compute_loss(self, batch): |
| 145 | losses = {} |
| 146 | losses["total"] = 0 |
| 147 | |
| 148 | # compute clip losses |
| 149 | batch = self.encode_text(batch) |
| 150 | batch = self.encode_motion(batch) |
| 151 | |
| 152 | mixed_clip_loss, clip_losses = self.compute_clip_losses(batch) |
| 153 | losses.update(clip_losses) |
nothing calls this directly
no outgoing calls
no test coverage detected