(self)
| 332 | nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) |
| 333 | |
| 334 | def build_attention_mask(self): |
| 335 | # lazily create causal attention mask, with full attention between the vision tokens |
| 336 | # pytorch uses additive attention mask; fill with -inf |
| 337 | mask = torch.empty(self.context_length, self.context_length) |
| 338 | mask.fill_(float("-inf")) |
| 339 | mask.triu_(1) # zero out the lower diagonal |
| 340 | return mask |
| 341 | |
| 342 | @property |
| 343 | def dtype(self): |