(self, config: Config)
| 696 | |
| 697 | class LLaMAMLP(lit_gpt.model.LLaMAMLP): |
| 698 | def __init__(self, config: Config) -> None: |
| 699 | nn.Module.__init__(self) |
| 700 | self.fc_1 = LoRALinear( |
| 701 | config.n_embd, |
| 702 | config.intermediate_size, |
| 703 | bias=config.bias, |
| 704 | r=(config.r if config.to_mlp else 0), |
| 705 | lora_alpha=config.alpha, |
| 706 | lora_dropout=config.dropout, |
| 707 | ) |
| 708 | self.fc_2 = LoRALinear( |
| 709 | config.n_embd, |
| 710 | config.intermediate_size, |
| 711 | bias=config.bias, |
| 712 | r=(config.r if config.to_mlp else 0), |
| 713 | lora_alpha=config.alpha, |
| 714 | lora_dropout=config.dropout, |
| 715 | ) |
| 716 | self.proj = LoRALinear( |
| 717 | config.intermediate_size, |
| 718 | config.n_embd, |
| 719 | bias=config.bias, |
| 720 | r=(config.r if config.to_mlp else 0), |
| 721 | lora_alpha=config.alpha, |
| 722 | lora_dropout=config.dropout, |
| 723 | ) |
| 724 | |
| 725 | def _load_from_state_dict( |
| 726 | self, state_dict: Dict, prefix: str, *args: Any, **kwargs: Any |
nothing calls this directly
no test coverage detected