| 33 | |
| 34 | |
| 35 | class FeedForwardNetwork(nn.Module): |
| 36 | |
| 37 | def __init__(self, input_dim, hidden_dim, output_dim): |
| 38 | super(FeedForwardNetwork, self).__init__() |
| 39 | self.layer1 = nn.Linear(input_dim, hidden_dim) |
| 40 | self.gelu = nn.GELU() |
| 41 | self.layer2 = nn.Linear(hidden_dim, output_dim) |
| 42 | |
| 43 | def forward(self, x): |
| 44 | x = self.layer1(x) |
| 45 | x = self.gelu(x) |
| 46 | x = self.layer2(x) |
| 47 | return x |
| 48 | |
| 49 | |
| 50 | class MultiHeadAttention(nn.Module): |