(self, x, *args, **kwargs)
| 307 | ) |
| 308 | # .to(x) so a quantized or fp8 base still works: the adapter weights |
| 309 | # meet the activation, not the other way around. |
| 310 | diff = bypass_diff( |
| 311 | x, |
| 312 | None, |
| 313 | self.lora_down.weight.to(x), |
| 314 | self.lora_up.weight.to(x), |
| 315 | self.lora_mid.weight.to(x) if self.tucker else None, |
| 316 | gamma=self.scale * scale, |
| 317 | extra_args=extra_args, |
| 318 | ) |
| 319 | scalar = self.scalar.to(device=diff.device, dtype=diff.dtype) |
| 320 | return self.dropout(diff * scalar) |
| 321 | |
| 322 | def bypass_forward(self, x, scale=1): |
| 323 | return self.org_forward(x) + self.bypass_forward_diff(x, scale=scale) |
| 324 | |
| 325 | def forward(self, x, *args, **kwargs): |
| 326 | if self.module_dropout and self.training: |
| 327 | if torch.rand(1) < self.module_dropout: |
| 328 | return self.org_forward(x, *args, **kwargs) |
| 329 | |
| 330 | if self.bypass_mode: |
| 331 | return self.bypass_forward(x, scale=self.multiplier) |
| 332 | |
| 333 | base = self.org_forward(x, *args, **kwargs) |
| 334 | scale = self.scale |
| 335 | device = x.device |
nothing calls this directly
no test coverage detected