| 57 | |
| 58 | |
| 59 | class FeedForward(nn.Module): |
| 60 | def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): |
| 61 | super().__init__() |
| 62 | inner_dim = int(dim * mult) |
| 63 | dim_out = default(dim_out, dim) |
| 64 | project_in = nn.Sequential( |
| 65 | nn.Linear(dim, inner_dim), |
| 66 | nn.GELU() |
| 67 | ) if not glu else GEGLU(dim, inner_dim) |
| 68 | |
| 69 | self.net = nn.Sequential( |
| 70 | project_in, |
| 71 | nn.Dropout(dropout), |
| 72 | nn.Linear(inner_dim, dim_out) |
| 73 | ) |
| 74 | |
| 75 | def forward(self, x): |
| 76 | return self.net(x) |
| 77 | |
| 78 | |
| 79 | def zero_module(module): |