(self, input, all_gather=False)
| 116 | return output |
| 117 | |
| 118 | def forward(self, input, all_gather=False): |
| 119 | # input |
| 120 | images = input['images'] |
| 121 | texts = input['captions'] |
| 122 | texts = self.sample_captions(texts) |
| 123 | # text&image encode |
| 124 | image_features = self.encode_image(images) |
| 125 | text_features = self.encode_text(texts) |
| 126 | |
| 127 | |
| 128 | # normalized features |
| 129 | image_features = image_features / (image_features.norm(dim=-1, keepdim=True)) |
| 130 | text_features = text_features / (text_features.norm(dim=-1, keepdim=True)+1e-10) |
| 131 | |
| 132 | # cosine similarity as logits |
| 133 | logit_scale = self.logit_scale.exp() |
| 134 | logit_scale.data = torch.clamp(logit_scale.data, max=100) |
| 135 | |
| 136 | if self.training and self.use_allgather or all_gather: |
| 137 | gathered_image_features = self.all_gather(image_features) |
| 138 | gathered_text_features = self.all_gather(text_features) |
| 139 | |
| 140 | logits_per_image = logit_scale * image_features @ gathered_text_features.t() |
| 141 | logits_per_text = logit_scale * text_features @ gathered_image_features.t() |
| 142 | else: |
| 143 | logits_per_image = logit_scale * image_features @ text_features.t() |
| 144 | logits_per_text = logit_scale * text_features @ image_features.t() |
| 145 | |
| 146 | return logits_per_image, logits_per_text |
| 147 | |
| 148 | |
| 149 | def clip_res50(**kwargs): |
nothing calls this directly
no test coverage detected