(self)
| 735 | self.mask = None |
| 736 | |
| 737 | def init_parameters(self): |
| 738 | if self.transformer is not None: |
| 739 | nn.init.normal_(self.token_embedding.weight, std=0.02) |
| 740 | nn.init.normal_(self.positional_embedding, std=0.01) |
| 741 | |
| 742 | proj_std = (self.transformer.width ** -0.5) * \ |
| 743 | ((2 * self.transformer.layers) ** -0.5) |
| 744 | attn_std = self.transformer.width ** -0.5 |
| 745 | fc_std = (2 * self.transformer.width) ** -0.5 |
| 746 | for block in self.transformer.resblocks: |
| 747 | nn.init.normal_(block.attn.in_proj_weight, std=attn_std) |
| 748 | nn.init.normal_(block.attn.out_proj.weight, std=proj_std) |
| 749 | nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) |
| 750 | nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) |
| 751 | |
| 752 | if self.text_projection is not None: |
| 753 | nn.init.normal_(self.text_projection, |
| 754 | std=self.transformer.width ** -0.5) |
| 755 | |
| 756 | def build_attention_mask(self): |
| 757 | # lazily create causal attention mask, with full attention between the vision tokens |
no outgoing calls
no test coverage detected