| 166 | |
| 167 | |
| 168 | def only_optimize_lora_parameters(model): |
| 169 | # turn off the gradient of all the parameters except the LoRA parameters |
| 170 | for name, param in model.named_parameters(): |
| 171 | if "lora_right_weight" in name or "lora_left_weight" in name: |
| 172 | param.requires_grad = True |
| 173 | else: |
| 174 | param.requires_grad = False |
| 175 | return model |
| 176 | |
| 177 | def make_model_gradient_checkpointing_compatible(model): |
| 178 | # Higgingface added this enable input require grads function to make gradient checkpointing work for lora-only optimization |