(self, input, return_dict=False)
| 107 | |
| 108 | |
| 109 | def forward(self, input, return_dict=False): |
| 110 | # data input |
| 111 | images = input['images'] |
| 112 | images_1, _ = torch.split(images, [3,3], dim=1) |
| 113 | texts = input['captions'] |
| 114 | texts = self.sample_captions(texts) |
| 115 | # text |
| 116 | text_features, word_features, text_labels = self.encode_text(texts, mask_type = self.text_mask_type) |
| 117 | # image |
| 118 | image_concat = images_1 |
| 119 | image_features_1, image_features_d = self.encode_image(image_concat, return_all=True) |
| 120 | # clip |
| 121 | logit_scale = self.logit_scale.exp() |
| 122 | image_features_1 = image_features_1 / (image_features_1.norm(dim=-1, keepdim=True)) |
| 123 | text_features = text_features / (text_features.norm(dim=-1, keepdim=True)+1e-10) |
| 124 | if self.training and self.use_allgather: |
| 125 | link.barrier() |
| 126 | gathered_image_features_1 = self.all_gather(image_features_1) |
| 127 | gathered_text_features = self.all_gather(text_features) |
| 128 | logits_per_image_1 = logit_scale * image_features_1 @ gathered_text_features.t() |
| 129 | logits_per_text_1 = logit_scale * text_features @ gathered_image_features_1.t() |
| 130 | # filip |
| 131 | if self.return_dense: |
| 132 | image_features_d1 = image_features_d |
| 133 | image_features_d1 = self.image_mapping(image_features_d1) |
| 134 | word_features_d = self.text_mapping(word_features) |
| 135 | logits_per_image_dense_1, logits_per_text_dense_1 = self.get_weighted_dense_logits(image_features_d1, word_features_d) |
| 136 | if return_dict: |
| 137 | ret_dict = {} |
| 138 | ret_dict['logits'] = logits_per_image_1, logits_per_text_1 |
| 139 | if self.return_dense: |
| 140 | ret_dict['dense_logits'] = logits_per_image_dense_1, logits_per_text_dense_1 |
| 141 | return ret_dict |
| 142 | raise NotImplementedError() |
| 143 | |
| 144 | |
| 145 |
nothing calls this directly
no test coverage detected