(model)
| 130 | |
| 131 | # convert the LoRA layer to linear layer |
| 132 | def convert_lora_to_linear_layer(model): |
| 133 | replace_name = [] |
| 134 | for name, module in model.named_modules(): |
| 135 | if isinstance(module, LinearLayer_LoRA): |
| 136 | replace_name.append(name) |
| 137 | for name in replace_name: |
| 138 | module = recursive_getattr(model, name) |
| 139 | zero_stage_3 = hasattr(module.weight, 'ds_id') |
| 140 | with deepspeed.zero.GatheredParameters(_z3_params_to_fetch([ |
| 141 | module.weight, module.bias, module.lora_left_weight, |
| 142 | module.lora_right_weight |
| 143 | ]), |
| 144 | modifier_rank=0, |
| 145 | enabled=zero_stage_3): |
| 146 | module.fuse_lora_weight() |
| 147 | return model |
| 148 | |
| 149 | def recover_lora(model): |
| 150 | replace_name = [] |
no test coverage detected