(model,
part_module_name,
lora_dim=0,
lora_scaling=1,
lora_dropout=0)
| 84 | |
| 85 | # convert the linear layer to LoRA |
| 86 | def convert_linear_layer_to_lora(model, |
| 87 | part_module_name, |
| 88 | lora_dim=0, |
| 89 | lora_scaling=1, |
| 90 | lora_dropout=0): |
| 91 | replace_name = [] |
| 92 | for name, module in model.named_modules(): |
| 93 | if isinstance(module, nn.Linear) and part_module_name in name: |
| 94 | replace_name.append(name) |
| 95 | for name in replace_name: |
| 96 | module = recursive_getattr(model, name) |
| 97 | tmp = LinearLayer_LoRA( |
| 98 | module.weight, lora_dim, lora_scaling, lora_dropout, |
| 99 | module.bias).to(module.weight.device).to(module.weight.dtype) |
| 100 | recursive_setattr(model, name, tmp) |
| 101 | return model |
| 102 | |
| 103 | |
| 104 | def mark_only_lora_as_trainable(model: nn.Module, bias: str = 'none') -> None: |
no test coverage detected