| 46 | |
| 47 | |
| 48 | class FeedForward(nn.Module): |
| 49 | def __init__(self, dim, hidden_dim): |
| 50 | super().__init__() |
| 51 | self.net = nn.Sequential( |
| 52 | nn.LayerNorm(dim), |
| 53 | BitLinear(dim, hidden_dim), |
| 54 | nn.GELU(), |
| 55 | BitLinear(hidden_dim, dim), |
| 56 | ) |
| 57 | |
| 58 | def forward(self, x): |
| 59 | return self.net(x) |
| 60 | |
| 61 | |
| 62 | class Transformer(nn.Module): |