(self, x: torch.Tensor)
| 425 | ) |
| 426 | |
| 427 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 428 | # x_fc_1 = self.fc_1(x) |
| 429 | # x_fc_2 = self.fc_2(x) |
| 430 | # x = torch.nn.functional.silu(x_fc_1) * x_fc_2 |
| 431 | # return self.proj(x) |
| 432 | return self.swiglu(x) |
| 433 | |
| 434 | |
| 435 | def build_rope_cache( |
nothing calls this directly
no outgoing calls
no test coverage detected