Method
__init__
(self,
layer_num,
r: int = 0,
lora_alpha: int = 1,
lora_dropout: float = 0.,
layer_range = None,
qlora = False,
cross_attention = True)
Source from the content-addressed store, hash-verified
| 180 | |
| 181 | class LoraMixin(BaseMixin): |
| 182 | def __init__(self, |
| 183 | layer_num, |
| 184 | r: int = 0, |
| 185 | lora_alpha: int = 1, |
| 186 | lora_dropout: float = 0., |
| 187 | layer_range = None, |
| 188 | qlora = False, |
| 189 | cross_attention = True): |
| 190 | super().__init__() |
| 191 | self.r = r |
| 192 | self.lora_alpha = lora_alpha |
| 193 | self.lora_dropout = lora_dropout |
| 194 | |
| 195 | if layer_range is None: |
| 196 | layer_range = [i for i in range(layer_num)] |
| 197 | self.layer_range = layer_range |
| 198 | |
| 199 | self.scaling = self.lora_alpha / self.r |
| 200 | self.qlora = qlora |
| 201 | self.cross_attention = cross_attention |
| 202 | |
| 203 | def reinit(self, parent_model): |
| 204 | for i in self.layer_range: |
Tested by
no test coverage detected