| 47 | |
| 48 | |
| 49 | class FeedForwardNet(nn.Module): |
| 50 | def __init__(self, channels: int, mlp_ratio: float = 4.0): |
| 51 | super().__init__() |
| 52 | self.mlp = nn.Sequential( |
| 53 | nn.Linear(channels, int(channels * mlp_ratio)), |
| 54 | nn.GELU(approximate="tanh"), |
| 55 | nn.Linear(int(channels * mlp_ratio), channels), |
| 56 | ) |
| 57 | |
| 58 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 59 | return self.mlp(x) |
| 60 | |
| 61 | |
| 62 | class TransformerBlock(nn.Module): |