Method forward

prototype/model/slip.py:245–286

(self, input, return_dict=False)

        return output

    def forward(self, input, return_dict=False):
        # input
        images = input['images']
        images_base, images_1, images_2 = torch.split(images, [3,3,3], dim=1)
        texts = input['captions']
        texts = self.sample_captions(texts)
        text_features = self.encode_text(texts)

        image_features = self.encode_image(images_base)
        image_features_1, image_sim_1 = self.encode_image(images_1, return_sim=True)
        image_features_2, image_sim_2 = self.encode_image(images_2, return_sim=True)

        # normalized features
        image_features = image_features / (image_features.norm(dim=-1, keepdim=True))
        text_features = text_features / (text_features.norm(dim=-1, keepdim=True)+1e-10)

        # image_sim_1 = image_sim_1 / (image_sim_1.norm(dim=-1, keepdim=True))
        # image_sim_2 = image_sim_2 / (image_sim_2.norm(dim=-1, keepdim=True))

        # cosine similarity as logits
        logit_scale = self.logit_scale.exp()
        if self.training and self.use_allgather:
            link.barrier()
            gathered_image_features = self.all_gather(image_features)
            gathered_text_features = self.all_gather(text_features)

            gathered_image_sim_1 = self.all_gather(image_sim_1)
            gathered_image_sim_2 = self.all_gather(image_sim_2)

            logits_per_image = logit_scale * image_features @ gathered_text_features.t()
            logits_per_text = logit_scale * text_features @ gathered_image_features.t()
        else:
            raise NotImplementedError('2-View: Not Implemented')

        if return_dict:
            link.barrier()
            ret_dict = {}
            ret_dict['logits'] = logits_per_image, logits_per_text
            ret_dict['sim_features'] = image_sim_1, gathered_image_sim_1, image_sim_2, gathered_image_sim_2
            ret_dict['features'] = text_features, image_features
            return ret_dict
        raise NotImplementedError('Must Return A Dict')


def slip_res50(**kwargs):