| 297 | self.initialize_parameters() |
| 298 | |
| 299 | def initialize_parameters(self): |
| 300 | nn.init.normal_(self.token_embedding.weight, std=0.02) |
| 301 | nn.init.normal_(self.positional_embedding, std=0.01) |
| 302 | |
| 303 | if isinstance(self.visual, ModifiedResNet): |
| 304 | if self.visual.attnpool is not None: |
| 305 | std = self.visual.attnpool.c_proj.in_features ** -0.5 |
| 306 | nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) |
| 307 | nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) |
| 308 | nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) |
| 309 | nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) |
| 310 | |
| 311 | for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]: |
| 312 | for name, param in resnet_block.named_parameters(): |
| 313 | if name.endswith("bn3.weight"): |
| 314 | nn.init.zeros_(param) |
| 315 | |
| 316 | proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) |
| 317 | attn_std = self.transformer.width ** -0.5 |
| 318 | fc_std = (2 * self.transformer.width) ** -0.5 |
| 319 | for block in self.transformer.resblocks: |
| 320 | nn.init.normal_(block.attn.in_proj_weight, std=attn_std) |
| 321 | nn.init.normal_(block.attn.out_proj.weight, std=proj_std) |
| 322 | nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) |
| 323 | nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) |
| 324 | |
| 325 | if self.text_projection is not None: |
| 326 | nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) |
| 327 | |
| 328 | def build_attention_mask(self): |
| 329 | # lazily create causal attention mask, with full attention between the vision tokens |