(self)
| 893 | self.init_weights() |
| 894 | |
| 895 | def init_weights(self) -> None: |
| 896 | std = 1.0 / math.sqrt(self.hidden_size) |
| 897 | torch.nn.init.trunc_normal_(self.linear.weight, std=std, a=-3 * std, b=3 * std) |
| 898 | if self.use_adaln_lora: |
| 899 | torch.nn.init.trunc_normal_(self.adaln_modulation[1].weight, std=std, a=-3 * std, b=3 * std) |
| 900 | torch.nn.init.zeros_(self.adaln_modulation[2].weight) |
| 901 | else: |
| 902 | torch.nn.init.zeros_(self.adaln_modulation[1].weight) |
| 903 | |
| 904 | self.layer_norm.reset_parameters() |
| 905 | |
| 906 | def forward( |
| 907 | self, |
no test coverage detected