(self, input, return_dict=False)
| 194 | |
| 195 | |
| 196 | def forward(self, input, return_dict=False): |
| 197 | # input |
| 198 | images = input['images'] |
| 199 | images_1, images_2 = torch.split(images, [3,3], dim=1) |
| 200 | texts = input['captions'] |
| 201 | #clip encode |
| 202 | texts = self.sample_captions(texts) |
| 203 | texts_aug = [] |
| 204 | for caption in texts: |
| 205 | if self.EDA: |
| 206 | emd_aug = choice([self.emd.synonym_replacement, self.emd.random_swap, self.emd.random_deletion]) |
| 207 | cap_new = emd_aug(caption) |
| 208 | if isinstance(cap_new, list): |
| 209 | cap_new = ' '.join(cap_new) |
| 210 | texts_aug.append(cap_new) # single word: there is a bug |
| 211 | else: |
| 212 | raise NotImplementedError('No EDA') |
| 213 | |
| 214 | if self.text_mask_type is not None: |
| 215 | text_features, word_features, text_labels = self.encode_text(texts, mask_type = self.text_mask_type) |
| 216 | text_features_aug = self.encode_text(texts_aug) |
| 217 | # text_features_aug, word_features_aug, text_labels_aug = self.encode_text(texts_aug, mask_type = self.text_mask_type) |
| 218 | else: |
| 219 | text_features = self.encode_text(texts) |
| 220 | if self.EDA: |
| 221 | text_features_aug = self.encode_text(texts_aug) |
| 222 | else: |
| 223 | text_features_aug = text_features.detach() |
| 224 | |
| 225 | if not self.return_dense: |
| 226 | if self.forward_type == 'image_concat': |
| 227 | image_concat = torch.cat([images_1, images_2], dim=0) |
| 228 | image_features_concat = self.encode_image(image_concat) |
| 229 | image_features_1, image_features_2 = torch.split(image_features_concat, images_1.shape[0], dim=0) |
| 230 | else: |
| 231 | image_features_1 = self.encode_image(images_1) |
| 232 | image_features_2 = self.encode_image(images_2) |
| 233 | else: |
| 234 | image_features_1, image_features_d1 = self.encode_image(images_1, return_dense=return_dense) |
| 235 | image_features_2, image_features_d2 = self.encode_image(images_2, return_dense=return_dense) |
| 236 | |
| 237 | #simsiam |
| 238 | z1 = self.projector(image_features_1) |
| 239 | z2 = self.projector(image_features_2) |
| 240 | p1 = self.predictor(z1) |
| 241 | p2 = self.predictor(z2) |
| 242 | |
| 243 | if self.return_dense: |
| 244 | b, n = image_features_d1.shape[:2] |
| 245 | #simsiam_dense |
| 246 | z1d = self.projector_d(image_features_d1.reshape(b*n, -1)) |
| 247 | z2d = self.projector_d(image_features_d2.reshape(b*n, -1)) |
| 248 | p1d = self.predictor_d(z1d).reshape(b,n,-1) |
| 249 | p2d = self.predictor_d(z2d).reshape(b,n,-1) |
| 250 | z1d, z2d = z1d.reshape(b,n,-1), z2d.reshape(b,n,-1) |
| 251 | |
| 252 | |
| 253 | # normalized features |
nothing calls this directly
no test coverage detected