(self)
| 828 | self.init_weights() |
| 829 | |
| 830 | def init_weights(self) -> None: |
| 831 | std = 1.0 / math.sqrt(self.dim) |
| 832 | torch.nn.init.trunc_normal_(self.proj[1].weight, std=std, a=-3 * std, b=3 * std) |
| 833 | |
| 834 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 835 | """ |