Initialize text encoder and related components.
(self)
| 133 | self.pixel_decoder = None |
| 134 | |
| 135 | def _init_text_components(self): |
| 136 | """Initialize text encoder and related components.""" |
| 137 | config = self.config |
| 138 | |
| 139 | act_layer = QuickGELU if config.text_quick_gelu else nn.GELU |
| 140 | norm_layer = LayerNorm |
| 141 | |
| 142 | text = TextTransformer( |
| 143 | context_length=config.text_context_length, |
| 144 | vocab_size=config.text_vocab_size, |
| 145 | width=config.text_embed_dim, |
| 146 | heads=config.text_num_heads, |
| 147 | layers=config.text_depth, |
| 148 | mlp_ratio=config.text_mlp_ratio, |
| 149 | ls_init_value=config.text_ls_init_value, |
| 150 | output_dim=config.text_embed_dim, |
| 151 | embed_cls=config.text_embed_cls, |
| 152 | no_causal_mask=config.text_no_causal_mask, |
| 153 | pad_id=config.text_pad_id, |
| 154 | pool_type=config.text_pool_type, |
| 155 | proj_type=config.text_proj_type, |
| 156 | proj_bias=config.text_proj_bias, |
| 157 | output_tokens=config.text_output_tokens, |
| 158 | act_layer=act_layer, |
| 159 | norm_layer=norm_layer, |
| 160 | ) |
| 161 | |
| 162 | self.text_transformer = text.transformer |
| 163 | self.context_length = text.context_length |
| 164 | self.vocab_size = text.vocab_size |
| 165 | self.token_embedding = text.token_embedding |
| 166 | self.positional_embedding = text.positional_embedding |
| 167 | self.ln_final = text.ln_final |
| 168 | self.text_projection = text.text_projection |
| 169 | self.text_pool_type = text.pool_type |
| 170 | self.register_buffer('attn_mask', text.attn_mask, persistent=False) |
| 171 | |
| 172 | # Logit scale |
| 173 | init_logit_scale = config.init_logit_scale or np.log(1 / 0.07) |
| 174 | lshape = [1] if config.nonscalar_logit_scale else [] |
| 175 | self.logit_scale = nn.Parameter(torch.ones(lshape) * init_logit_scale) |
| 176 | |
| 177 | if config.init_logit_bias is not None: |
| 178 | self.logit_bias = nn.Parameter(torch.ones(lshape) * config.init_logit_bias) |
| 179 | else: |
| 180 | self.logit_bias = None |
| 181 | |
| 182 | # ==================== Basic Feature Methods ==================== |
| 183 |
no test coverage detected