| 17 | return self |
| 18 | |
| 19 | def __init__(self, args, hidden_size): |
| 20 | super().__init__() |
| 21 | self.n_learnable_queries = 32 |
| 22 | config = AutoConfig.from_pretrained(args.text_condition) |
| 23 | self.multi_encoder = AutoModel.from_config(config) |
| 24 | qformer_config = Blip2QFormerConfig( |
| 25 | num_hidden_layers=12, |
| 26 | encoder_hidden_size=self.multi_encoder.config.text_config.hidden_size |
| 27 | ) |
| 28 | self.qformer = Blip2QFormerModel(qformer_config) |
| 29 | self.query_embeds = nn.Embedding(self.n_learnable_queries, qformer_config.hidden_size) |
| 30 | self.out_project = nn.Linear(qformer_config.hidden_size, hidden_size) |
| 31 | |
| 32 | @torch.no_grad() |
| 33 | def encode_text(self, input_ids, attention_mask): |