| 23 | |
| 24 | class ImagePatchEmbeddingMixin(BaseMixin): |
| 25 | def __init__( |
| 26 | self, |
| 27 | in_channels, |
| 28 | hidden_size, # * 1920 |
| 29 | patch_size, |
| 30 | bias=True, |
| 31 | text_hidden_size=None, |
| 32 | cond_emb_in_dim=None, # * Indicating conditional frames, related to t |
| 33 | use_aug_cond_emb=False, # * Indicating the scale of noise for conditioning augmentation, related to t_aug |
| 34 | ): |
| 35 | super().__init__() |
| 36 | self.proj = nn.Conv2d(in_channels, hidden_size, kernel_size=patch_size, stride=patch_size, bias=bias) |
| 37 | if text_hidden_size is not None: |
| 38 | self.text_proj = nn.Linear(text_hidden_size, hidden_size) |
| 39 | else: |
| 40 | self.text_proj = None |
| 41 | |
| 42 | if cond_emb_in_dim is not None: |
| 43 | self.cond_emb_proj = nn.Linear(cond_emb_in_dim, hidden_size) |
| 44 | # * zero_init |
| 45 | nn.init.constant_(self.cond_emb_proj.weight, 0) |
| 46 | nn.init.constant_(self.cond_emb_proj.bias, 0) |
| 47 | else: |
| 48 | self.cond_emb_proj = None |
| 49 | |
| 50 | if use_aug_cond_emb: |
| 51 | self.aug_cond_emb_proj = nn.Linear(cond_emb_in_dim, hidden_size) # aug_cond_emb is of the same shape with cond_emb |
| 52 | # * zero_init |
| 53 | nn.init.constant_(self.aug_cond_emb_proj.weight, 0) |
| 54 | nn.init.constant_(self.aug_cond_emb_proj.bias, 0) |
| 55 | else: |
| 56 | self.aug_cond_emb_proj = None |
| 57 | |
| 58 | def word_embedding_forward(self, input_ids, **kwargs): |
| 59 | # now is 3d patch |