(self, rank: int = 4, lora_alpha: int = 1, *args, **kwargs)
| 48 | """ |
| 49 | |
| 50 | def __init__(self, rank: int = 4, lora_alpha: int = 1, *args, **kwargs): |
| 51 | super(BitLora, self).__init__(*args, **kwargs) |
| 52 | self.rank = rank |
| 53 | self.lora_alpha = lora_alpha |
| 54 | self.scaling = self.lora_alpha / self.rank |
| 55 | self.merged = False |
| 56 | |
| 57 | self.lora_a = nn.Parameter(torch.zeros(self.in_features, rank)) |
| 58 | self.lora_b = nn.Parameter(torch.zeros(rank, self.out_features)) |
| 59 | |
| 60 | # Rmsnorm |
| 61 | self.rms_norm = SimpleRMSNorm(self.in_features) |
| 62 | |
| 63 | def forward(self, x: Tensor) -> Tensor: |
| 64 | """ |
nothing calls this directly
no outgoing calls
no test coverage detected