| 355 | self.initialize_parameters() |
| 356 | |
| 357 | def initialize_parameters(self): |
| 358 | nn.init.normal_(self.token_embedding.weight, std=0.02) |
| 359 | nn.init.normal_(self.positional_embedding, std=0.01) |
| 360 | |
| 361 | if isinstance(self.visual, ModifiedResNet): |
| 362 | if self.visual.attnpool is not None: |
| 363 | std = self.visual.attnpool.c_proj.in_features ** -0.5 |
| 364 | nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) |
| 365 | nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) |
| 366 | nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) |
| 367 | nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) |
| 368 | |
| 369 | for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]: |
| 370 | for name, param in resnet_block.named_parameters(): |
| 371 | if name.endswith("bn3.weight"): |
| 372 | nn.init.zeros_(param) |
| 373 | |
| 374 | proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) |
| 375 | attn_std = self.transformer.width ** -0.5 |
| 376 | fc_std = (2 * self.transformer.width) ** -0.5 |
| 377 | for block in self.transformer.resblocks: |
| 378 | nn.init.normal_(block.attn.in_proj_weight, std=attn_std) |
| 379 | nn.init.normal_(block.attn.out_proj.weight, std=proj_std) |
| 380 | nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) |
| 381 | nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) |
| 382 | |
| 383 | if self.text_projection is not None: |
| 384 | nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) |
| 385 | |
| 386 | def build_attention_mask(self): |
| 387 | # lazily create causal attention mask, with full attention between the vision tokens |