(
self,
lora_name,
org_module: nn.Module,
multiplier=1.0,
lora_dim=4,
alpha=1,
dropout=0.0,
rank_dropout=0.0,
module_dropout=0.0,
use_tucker=False,
use_scalar=False,
rank_dropout_scale=False,
weight_decompose=False,
wd_on_out=True,
bypass_mode=None,
rs_lora=False,
**kwargs,
)
| 28 | "dora_scale", |
| 29 | ] |
| 30 | weight_list_det = ["hada_w1_a"] |
| 31 | |
| 32 | def __init__( |
| 33 | self, |
| 34 | lora_name, |
| 35 | org_module: nn.Module, |
| 36 | multiplier=1.0, |
| 37 | lora_dim=4, |
| 38 | alpha=1, |
| 39 | dropout=0.0, |
| 40 | rank_dropout=0.0, |
| 41 | module_dropout=0.0, |
| 42 | use_tucker=False, |
| 43 | use_scalar=False, |
| 44 | rank_dropout_scale=False, |
| 45 | weight_decompose=False, |
| 46 | wd_on_out=True, |
| 47 | bypass_mode=None, |
| 48 | rs_lora=False, |
| 49 | **kwargs, |
| 50 | ): |
| 51 | super().__init__( |
| 52 | lora_name, |
| 53 | org_module, |
| 54 | multiplier, |
| 55 | dropout, |
| 56 | rank_dropout, |
| 57 | module_dropout, |
| 58 | rank_dropout_scale, |
| 59 | bypass_mode, |
| 60 | ) |
| 61 | if self.module_type not in self.support_module: |
| 62 | raise ValueError(f"{self.module_type} is not supported in LoHa algo.") |
| 63 | self.lora_name = lora_name |
| 64 | self.lora_dim = lora_dim |
| 65 | self.tucker = False |
| 66 | self.rs_lora = rs_lora |
| 67 | |
| 68 | w_shape = self.shape |
| 69 | if self.module_type.startswith("conv"): |
| 70 | in_dim = org_module.in_channels // org_module.groups |
| 71 | k_size = org_module.kernel_size |
| 72 | out_dim = org_module.out_channels |
| 73 | self.shape = (out_dim, in_dim, *k_size) |
| 74 | self.tucker = use_tucker and any(i != 1 for i in k_size) |
| 75 | if self.tucker: |
| 76 | w_shape = (out_dim, in_dim, *k_size) |
| 77 | else: |
| 78 | w_shape = (out_dim, in_dim * torch.tensor(k_size).prod().item()) |
| 79 | |
| 80 | if self.tucker: |
| 81 | self.hada_t1 = nn.Parameter(torch.empty(lora_dim, lora_dim, *w_shape[2:])) |
| 82 | self.hada_w1_a = nn.Parameter( |
| 83 | torch.empty(lora_dim, w_shape[0]) |
| 84 | ) # out_dim, 1-mode |
| 85 | self.hada_w1_b = nn.Parameter( |
| 86 | torch.empty(lora_dim, w_shape[1]) |
| 87 | ) # in_dim , 2-mode |
nothing calls this directly
no outgoing calls
no test coverage detected