(self, device=None)
| 196 | if isinstance(self.scalar, nn.Parameter): |
| 197 | self.scalar.data.copy_(torch.ones_like(self.scalar)) |
| 198 | elif getattr(self, "scalar", None) is not None: |
| 199 | self.scalar.copy_(torch.ones_like(self.scalar)) |
| 200 | else: |
| 201 | self.register_buffer( |
| 202 | "scalar", torch.ones_like(self.scalar), persistent=False |
| 203 | ) |
| 204 | |
| 205 | def make_weight(self, device=None): |
| 206 | wa = self.lora_up.weight.to(device) |
| 207 | wb = self.lora_down.weight.to(device) |
| 208 | t = self.lora_mid.weight.to(device) if self.tucker else None |
| 209 | # gamma=1: self.scale belongs to the caller and self.scalar can be a |
| 210 | # parameter, so neither folds into the rebuild's scale. |
| 211 | weight = diff_weight(wb, wa, t, gamma=1.0).view(self.shape) |
| 212 | if self.training and self.rank_dropout: |
| 213 | drop = (torch.rand(weight.size(0), device=device) > self.rank_dropout).to( |
| 214 | weight.dtype |
| 215 | ) |
| 216 | drop = drop.view(-1, *[1] * len(weight.shape[1:])) |
| 217 | if self.rank_dropout_scale: |
| 218 | drop /= drop.mean() |
| 219 | weight *= drop |
| 220 | |
| 221 | return weight * self.scalar.to(device) |
| 222 |
no test coverage detected