| 11 | |
| 12 | |
| 13 | class LanguageEncoder(nn.Module): |
| 14 | |
| 15 | @configurable |
| 16 | def __init__( |
| 17 | self, |
| 18 | tokenizer, |
| 19 | tokenizer_type, |
| 20 | lang_encoder, |
| 21 | lang_projection, |
| 22 | max_token_num, |
| 23 | ): |
| 24 | super().__init__() |
| 25 | self.tokenizer = tokenizer |
| 26 | self.tokenizer_type = tokenizer_type |
| 27 | self.lang_encoder = lang_encoder |
| 28 | self.lang_proj = lang_projection |
| 29 | self.max_token_num = max_token_num |
| 30 | self.logit_scale = nn.Parameter(torch.ones([])) |
| 31 | |
| 32 | @classmethod |
| 33 | def from_config(cls, cfg): |
| 34 | # build up text encoder |
| 35 | tokenizer = build_tokenizer(cfg['MODEL']['TEXT']) |
| 36 | tokenizer_type = cfg['MODEL']['TEXT']['TOKENIZER'] |
| 37 | lang_encoder = build_lang_encoder(cfg['MODEL']['TEXT'], tokenizer, cfg['VERBOSE']) |
| 38 | max_token_num = cfg['MODEL']['TEXT']['CONTEXT_LENGTH'] |
| 39 | |
| 40 | dim_lang = cfg['MODEL']['TEXT']['WIDTH'] |
| 41 | dim_projection = cfg['MODEL']['DIM_PROJ'] |
| 42 | lang_projection = nn.Parameter(torch.empty(dim_lang, dim_projection)) |
| 43 | trunc_normal_(lang_projection, std=.02) |
| 44 | |
| 45 | return { |
| 46 | "tokenizer": tokenizer, |
| 47 | "tokenizer_type": tokenizer_type, |
| 48 | "lang_encoder": lang_encoder, |
| 49 | "lang_projection": lang_projection, |
| 50 | "max_token_num": max_token_num, |
| 51 | } |
| 52 | |
| 53 | # @torch.no_grad() |
| 54 | def get_text_embeddings(self, class_names, name='default', is_eval=False, add_bgd=False, prompt=True, norm=True): |
| 55 | if not is_eval: |
| 56 | if prompt: |
| 57 | # randomly sample one template |
| 58 | arbitary_concepts = [ |
| 59 | prompt_engineering(class_names[label].replace('-other','').replace('-merged','').replace('-stuff',''), topk=10000, suffix='.') \ |
| 60 | for label in range(len(class_names)) |
| 61 | ] |
| 62 | if add_bgd: |
| 63 | arbitary_concepts.append("A background in coco.") |
| 64 | else: |
| 65 | arbitary_concepts = class_names |
| 66 | |
| 67 | input_ids = [] |
| 68 | attention_masks = [] |
| 69 | for txt in arbitary_concepts: |
| 70 | tokens = self.tokenizer( |
no outgoing calls
no test coverage detected