| 11 | |
| 12 | @BACKBONE_REGISTRY.register() |
| 13 | class CLIPLangEncoder(Backbone): |
| 14 | def __init__(self, cfg, input_shape): |
| 15 | super().__init__() |
| 16 | |
| 17 | text_cfg = cfg.MODEL.TEXT |
| 18 | |
| 19 | model_name = text_cfg.MODEL_NAME |
| 20 | token_name = text_cfg.TOKEN_NAME |
| 21 | |
| 22 | self.tokenizer = AutoTokenizer.from_pretrained(token_name) |
| 23 | clip_config = CLIPTextConfig.from_pretrained(model_name) |
| 24 | # attention_dropout=dropout) |
| 25 | if 'CLIP-ViT-L-14-laion2B-s32B-b82K' in model_name: |
| 26 | clip_config.projection_dim = 768 |
| 27 | |
| 28 | # self.model = CLIPTP.from_pretrained(model_name, config=clip_config) |
| 29 | self.model = CLIPTP(config=clip_config) |
| 30 | |
| 31 | def forward(self, text): |
| 32 | num_per_batch = [len(t) for t in text] |
| 33 | assert max(num_per_batch) == min(num_per_batch), ( |
| 34 | 'number of sequences not equal in batch') |
| 35 | text = list(itertools.chain(*text)) |
| 36 | text = self.tokenizer(text=text, return_tensors='pt', padding=True, max_length=20) |
| 37 | text = text.to(device=self.model.device) |
| 38 | txt_outputs = self.model(**text) |
| 39 | txt_feats = txt_outputs.text_embeds |
| 40 | txt_feats = txt_feats.reshape(-1, num_per_batch[0], |
| 41 | txt_feats.shape[-1]) |
| 42 | return txt_feats |
nothing calls this directly
no outgoing calls
no test coverage detected