(self, hidden_states, lora_layer_params=None)
| 389 | return inter |
| 390 | |
| 391 | def fc_gate(self, hidden_states, lora_layer_params=None): |
| 392 | # Combine the following pattern |
| 393 | # |
| 394 | # SiLU(FC(x)) * Gate(x) |
| 395 | # |
| 396 | # into: |
| 397 | # |
| 398 | # SwiGLU(FusedFC(x)) |
| 399 | # |
| 400 | # Upside is we don't need to modify 4 different weight loading paths just to concat weights |
| 401 | |
| 402 | inter = self.fused_fc(hidden_states) |
| 403 | |
| 404 | lora_result = fc_gate_lora(hidden_states, self.lora, |
| 405 | self.fused_gate_up_lora, lora_layer_params) |
| 406 | if lora_result is not None: |
| 407 | inter = inter + lora_result |
| 408 | if self.dora is not None: |
| 409 | inter = fc_gate_dora(inter, self.dora, self.fused_gate_up_lora, |
| 410 | lora_layer_params) |
| 411 | |
| 412 | if self.hidden_act == 'silu': |
| 413 | inter = ACT2FN['swiglu'](inter) |
| 414 | elif self.hidden_act == 'gelu': |
| 415 | inter = ACT2FN['geglu'](inter) |
| 416 | else: |
| 417 | raise NotImplementedError( |
| 418 | f"Activation {self.hidden_act} not yet implemented for {self.__class__.__name__}." |
| 419 | ) |
| 420 | return inter |
| 421 | |
| 422 | def forward(self, |
| 423 | hidden_states, |
no test coverage detected