| 64 | self.init_weights(add_constant) |
| 65 | |
| 66 | def init_weights(self, add_constant: bool = False): |
| 67 | def basic_init(module): |
| 68 | if isinstance(module, nn.Linear): |
| 69 | nn.init.xavier_uniform_(module.weight) |
| 70 | if module.bias is not None: |
| 71 | nn.init.constant_(module.bias, 0) |
| 72 | self.apply(basic_init) |
| 73 | |
| 74 | # For no pre-optimized training, you should consider use the following init |
| 75 | # with self.down = down@down_aux + 1 in LiLoRAAttnProcessor |
| 76 | # if add_constant: |
| 77 | torch.nn.init.constant_(self.delta_proj[1].weight, 0) |
| 78 | |
| 79 | # advice from Nataniel Ruiz, looks like 1e-3 is small enough |
| 80 | # else: |
| 81 | # torch.nn.init.normal_(self.delta_proj[1].weight, std=1e-3) |
| 82 | |
| 83 | def forward(self, weight, features): |
| 84 | pos_emb = self.pos_emb_proj(self.block_pos_emb[:, :weight.size(1)].clone().detach()) |