if alpha == 0 or None, alpha is rank (no scaling).
(
self,
lora_name,
org_module: nn.Module,
multiplier=1.0,
lora_dim=4,
alpha=1,
dropout=0.0,
rank_dropout=0.0,
module_dropout=0.0,
use_tucker=False,
use_scalar=False,
rank_dropout_scale=False,
weight_decompose=False,
wd_on_out=True,
bypass_mode=None,
rs_lora=False,
**kwargs,
)
| 36 | ] |
| 37 | weight_list_det = ["lora_up.weight"] |
| 38 | |
| 39 | def __init__( |
| 40 | self, |
| 41 | lora_name, |
| 42 | org_module: nn.Module, |
| 43 | multiplier=1.0, |
| 44 | lora_dim=4, |
| 45 | alpha=1, |
| 46 | dropout=0.0, |
| 47 | rank_dropout=0.0, |
| 48 | module_dropout=0.0, |
| 49 | use_tucker=False, |
| 50 | use_scalar=False, |
| 51 | rank_dropout_scale=False, |
| 52 | weight_decompose=False, |
| 53 | wd_on_out=True, |
| 54 | bypass_mode=None, |
| 55 | rs_lora=False, |
| 56 | **kwargs, |
| 57 | ): |
| 58 | """if alpha == 0 or None, alpha is rank (no scaling).""" |
| 59 | super().__init__( |
| 60 | lora_name, |
| 61 | org_module, |
| 62 | multiplier, |
| 63 | dropout, |
| 64 | rank_dropout, |
| 65 | module_dropout, |
| 66 | rank_dropout_scale, |
| 67 | bypass_mode, |
| 68 | ) |
| 69 | if self.module_type not in self.support_module: |
| 70 | raise ValueError(f"{self.module_type} is not supported in LoRA/LoCon algo.") |
| 71 | self.lora_dim = lora_dim |
| 72 | self.tucker = False |
| 73 | self.rs_lora = rs_lora |
| 74 | |
| 75 | if self.module_type.startswith("conv"): |
| 76 | self.isconv = True |
| 77 | # For general LoCon. in_dim follows torch Conv weight layout (in/groups) |
| 78 | # so rebuild_weight matches F.conv*d when groups != 1 (#260). |
| 79 | in_dim = org_module.in_channels // org_module.groups |
| 80 | k_size = org_module.kernel_size |
| 81 | stride = org_module.stride |
| 82 | padding = org_module.padding |
| 83 | out_dim = org_module.out_channels |
| 84 | use_tucker = use_tucker and any(i != 1 for i in k_size) |
| 85 | self.down_op = self.op |
| 86 | self.up_op = self.op |
| 87 | if org_module.groups != 1 and self.bypass_mode: |
| 88 | # Adapter Conv modules are groups=1 and take in/groups channels; |
| 89 | # bypass forward on the full activation is not valid for grouped |
| 90 | # originals, so force the weight-rebuild path. |
| 91 | self.bypass_mode = False |
| 92 | if use_tucker and any(i != 1 for i in k_size): |
| 93 | self.lora_down = self.module(in_dim, lora_dim, 1, bias=False) |
| 94 | self.lora_mid = self.module( |
| 95 | lora_dim, lora_dim, k_size, stride, padding, bias=False |