(self)
| 123 | requires_grad=False) |
| 124 | |
| 125 | def init_lora(self): |
| 126 | if self.disabled: |
| 127 | return |
| 128 | |
| 129 | if self.quantization_config is not None: |
| 130 | # ensure quant-param wasn't stripped, in some cases transformers will do this during model init |
| 131 | if not isinstance(self.weight, QuantizedParameter): |
| 132 | self.weight = QuantizedParameter(self.weight, quantization_config=self.quantization_config) |
| 133 | |
| 134 | self._initialized = True |
| 135 | self.weight.requires_grad = False |
| 136 | |
| 137 | # Mark base weight to prevent broadcast and ensure proper offload behavior |
| 138 | self.weight.ds_optim_param = True |
| 139 | |
| 140 | self.lora_scaling_factor = self.lora_config.lora_alpha / self.lora_config.lora_r |
| 141 | |
| 142 | # Keeping lora weights in bf16 precision for ease of training. |
| 143 | self.lora_weight_1 = self.linear_cls(self.input_dim, |
| 144 | self.lora_config.lora_r, |
| 145 | bias=self.bias, |
| 146 | device=self.device, |
| 147 | dtype=self.dtype) |
| 148 | self.lora_weight_2 = self.linear_cls(self.lora_config.lora_r, |
| 149 | self.output_dim, |
| 150 | bias=self.bias, |
| 151 | device=self.device, |
| 152 | dtype=self.dtype) |
| 153 | |
| 154 | # initialize "A" with kaiming uniform and "B" with zeros following this |
| 155 | # https://github.com/huggingface/peft/blob/62122b5add8d6892f70c82eaef2147a6ba33b90b/src/peft/tuners/lora/layer.py#L155 |
| 156 | nn.init.kaiming_uniform_(self.lora_weight_1.weight, a=math.sqrt(5)) |
| 157 | nn.init.zeros_(self.lora_weight_2.weight) |
| 158 | self.lora_weight_1.weight.requires_grad = True |
| 159 | self.lora_weight_2.weight.requires_grad = True |
| 160 | |
| 161 | def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, |
| 162 | error_msgs): |
no test coverage detected