(self, n_embd: int, dropout: float = 0.0)
| 56 | |
| 57 | class MLP(nn.Module): |
| 58 | def __init__(self, n_embd: int, dropout: float = 0.0): |
| 59 | super().__init__() |
| 60 | self.c_fc = nn.Linear(n_embd, 4 * n_embd) |
| 61 | self.gelu = nn.GELU() |
| 62 | self.c_proj = nn.Linear(4 * n_embd, n_embd) |
| 63 | self.dropout = nn.Dropout(dropout) |
| 64 | |
| 65 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 66 | x = self.c_fc(x) |