Initialize Linear. Args: in_features: TODO. out_features: TODO. r: TODO. lora_alpha: TODO. lora_dropout: TODO. fan_in_fan_out: TODO. merge_weights: TODO.
(
self,
in_features: int,
out_features: int,
r: int = 0,
lora_alpha: int = 1,
lora_dropout: float = 0.0,
fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out)
merge_weights: bool = True,
**kwargs
)
| 129 | class Linear(nn.Linear, LoRALayer): |
| 130 | # LoRA implemented in a dense layer |
| 131 | def __init__( |
| 132 | self, |
| 133 | in_features: int, |
| 134 | out_features: int, |
| 135 | r: int = 0, |
| 136 | lora_alpha: int = 1, |
| 137 | lora_dropout: float = 0.0, |
| 138 | fan_in_fan_out: bool = False, # Set this to True if the layer to replace stores weight like (fan_in, fan_out) |
| 139 | merge_weights: bool = True, |
| 140 | **kwargs |
| 141 | ): |
| 142 | """Initialize Linear. |
| 143 | |
| 144 | Args: |
| 145 | in_features: TODO. |
| 146 | out_features: TODO. |
| 147 | r: TODO. |
| 148 | lora_alpha: TODO. |
| 149 | lora_dropout: TODO. |
| 150 | fan_in_fan_out: TODO. |
| 151 | merge_weights: TODO. |
| 152 | **kwargs: Additional keyword arguments. |
| 153 | """ |
| 154 | nn.Linear.__init__(self, in_features, out_features, **kwargs) |
| 155 | LoRALayer.__init__( |
| 156 | self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights |
| 157 | ) |
| 158 | |
| 159 | self.fan_in_fan_out = fan_in_fan_out |
| 160 | # Actual trainable parameters |
| 161 | if r > 0: |
| 162 | self.lora_A = nn.Parameter(self.weight.new_zeros((r, in_features))) |
| 163 | self.lora_B = nn.Parameter(self.weight.new_zeros((out_features, r))) |
| 164 | self.scaling = self.lora_alpha / self.r |
| 165 | # Freezing the pre-trained weight matrix |
| 166 | self.weight.requires_grad = False |
| 167 | self.reset_parameters() |
| 168 | if fan_in_fan_out: |
| 169 | self.weight.data = self.weight.data.T |
| 170 | |
| 171 | def reset_parameters(self): |
| 172 | """Reset parameters.""" |
no test coverage detected