| 192 | |
| 193 | class Basic3DPositionEmbeddingMixin(BaseMixin): |
| 194 | def __init__( |
| 195 | self, |
| 196 | height, |
| 197 | width, |
| 198 | compressed_num_frames, |
| 199 | hidden_size, |
| 200 | text_length=0, |
| 201 | height_interpolation=1.0, |
| 202 | width_interpolation=1.0, |
| 203 | time_interpolation=1.0, |
| 204 | ): |
| 205 | super().__init__() |
| 206 | self.height = height |
| 207 | self.width = width |
| 208 | self.text_length = text_length |
| 209 | self.compressed_num_frames = compressed_num_frames |
| 210 | self.spatial_length = height * width |
| 211 | self.num_patches = height * width * compressed_num_frames |
| 212 | self.pos_embedding = nn.Parameter( |
| 213 | torch.zeros(1, int(text_length + self.num_patches), int(hidden_size)), requires_grad=False |
| 214 | ) # * requires_grad=False, won't be updated. |
| 215 | self.height_interpolation = height_interpolation |
| 216 | self.width_interpolation = width_interpolation |
| 217 | self.time_interpolation = time_interpolation |
| 218 | |
| 219 | def position_embedding_forward(self, position_ids, **kwargs): |
| 220 | if kwargs["images"].shape[1] == 1: |