(self, input, return_dict=False)
| 270 | |
| 271 | |
| 272 | def forward(self, input, return_dict=False): |
| 273 | # input |
| 274 | images = input['images'] |
| 275 | images_1, images_2 = torch.split(images, [3,3], dim=1) |
| 276 | texts = input['captions'] |
| 277 | #clip encode |
| 278 | texts = self.sample_captions(texts) |
| 279 | texts_aug = [] |
| 280 | for caption in texts: |
| 281 | if self.EDA: |
| 282 | emd_aug = choice([self.emd.synonym_replacement, self.emd.random_swap, self.emd.random_deletion]) |
| 283 | cap_new = emd_aug(caption) |
| 284 | if isinstance(cap_new, list): |
| 285 | cap_new = ' '.join(cap_new) |
| 286 | texts_aug.append(cap_new) # single word: there is a bug |
| 287 | else: |
| 288 | raise NotImplementedError('No EDA') |
| 289 | |
| 290 | if self.text_mask_type is not None: |
| 291 | text_features, word_features, text_labels = self.encode_text(texts, mask_type = self.text_mask_type) |
| 292 | text_features_aug, word_features_aug = self.encode_text(texts_aug, return_dense=True) |
| 293 | # text_features_aug, word_features_aug, text_labels_aug = self.encode_text(texts_aug, mask_type = self.text_mask_type) |
| 294 | else: |
| 295 | text_features = self.encode_text(texts) |
| 296 | if self.EDA: |
| 297 | text_features_aug = self.encode_text(texts_aug) |
| 298 | else: |
| 299 | text_features_aug = text_features.detach() |
| 300 | |
| 301 | if not self.return_filip: |
| 302 | if self.forward_type == 'image_concat': |
| 303 | image_concat = torch.cat([images_1, images_2], dim=0) |
| 304 | image_features_concat = self.encode_image(image_concat) |
| 305 | image_features_1, image_features_2 = torch.split(image_features_concat, images_1.shape[0], dim=0) |
| 306 | else: |
| 307 | image_features_1 = self.encode_image(images_1) |
| 308 | image_features_2 = self.encode_image(images_2) |
| 309 | else: |
| 310 | return_dense = True |
| 311 | image_features_1, image_features_d1 = self.encode_image(images_1, return_dense=return_dense) |
| 312 | image_features_2, image_features_d2 = self.encode_image(images_2, return_dense=return_dense) |
| 313 | |
| 314 | #simsiam |
| 315 | z1 = self.projector(image_features_1) |
| 316 | z2 = self.projector(image_features_2) |
| 317 | p1 = self.predictor(z1) |
| 318 | p2 = self.predictor(z2) |
| 319 | |
| 320 | if self.return_dense: |
| 321 | b, n = image_features_d1.shape[:2] |
| 322 | #simsiam_dense |
| 323 | z1d = self.projector_d(image_features_d1.reshape(b*n, -1)) |
| 324 | z2d = self.projector_d(image_features_d2.reshape(b*n, -1)) |
| 325 | p1d = self.predictor_d(z1d).reshape(b,n,-1) |
| 326 | p2d = self.predictor_d(z2d).reshape(b,n,-1) |
| 327 | z1d, z2d = z1d.reshape(b,n,-1), z2d.reshape(b,n,-1) |
| 328 | |
| 329 | if self.return_filip: |
nothing calls this directly
no test coverage detected