(
self,
lora_name,
org_module: nn.Module,
multiplier=1.0,
lora_dim=4,
alpha=1,
dropout=0.0,
rank_dropout=0.0,
module_dropout=0.0,
use_tucker=False,
use_scalar=False,
rank_dropout_scale=False,
bypass_mode=None,
**kwargs,
)
| 28 | weight_list = ["diff", "diff_b"] |
| 29 | weight_list_det = ["diff"] |
| 30 | |
| 31 | def __init__( |
| 32 | self, |
| 33 | lora_name, |
| 34 | org_module: nn.Module, |
| 35 | multiplier=1.0, |
| 36 | lora_dim=4, |
| 37 | alpha=1, |
| 38 | dropout=0.0, |
| 39 | rank_dropout=0.0, |
| 40 | module_dropout=0.0, |
| 41 | use_tucker=False, |
| 42 | use_scalar=False, |
| 43 | rank_dropout_scale=False, |
| 44 | bypass_mode=None, |
| 45 | **kwargs, |
| 46 | ): |
| 47 | org_bypass = bypass_mode |
| 48 | super().__init__( |
| 49 | lora_name, |
| 50 | org_module, |
| 51 | multiplier, |
| 52 | dropout, |
| 53 | rank_dropout, |
| 54 | module_dropout, |
| 55 | rank_dropout_scale, |
| 56 | bypass_mode, |
| 57 | ) |
| 58 | if bypass_mode and org_bypass is None: |
| 59 | self.bypass_mode = False |
| 60 | log_bypass_override() |
| 61 | |
| 62 | if self.module_type not in self.support_module: |
| 63 | raise ValueError(f"{self.module_type} is not supported in Full algo.") |
| 64 | |
| 65 | if self.is_quant: |
| 66 | raise ValueError( |
| 67 | "Quant Linear is not supported and meaningless in Full algo." |
| 68 | ) |
| 69 | |
| 70 | if self.bypass_mode: |
| 71 | raise ValueError("bypass mode is not supported in Full algo.") |
| 72 | |
| 73 | self.weight = nn.Parameter(torch.zeros_like(org_module.weight)) |
| 74 | if org_module.bias is not None: |
| 75 | self.bias = nn.Parameter(torch.zeros_like(org_module.bias)) |
| 76 | else: |
| 77 | self.bias = None |
| 78 | self.is_diff = True |
| 79 | self._org_weight = [self.org_module[0].weight.data.cpu().clone()] |
| 80 | if self.org_module[0].bias is not None: |
| 81 | self.org_bias = [self.org_module[0].bias.data.cpu().clone()] |
| 82 | else: |
| 83 | self.org_bias = None |
| 84 | |
| 85 | @classmethod |
nothing calls this directly
no test coverage detected