| 337 | |
| 338 | |
| 339 | class FeedForward(nn.Module): |
| 340 | def __init__(self, dim, dim_out=None, mult=4, glu=False, dropout=0.): |
| 341 | super().__init__() |
| 342 | inner_dim = int(dim * mult) |
| 343 | dim_out = default(dim_out, dim) |
| 344 | project_in = nn.Sequential( |
| 345 | nn.Linear(dim, inner_dim), |
| 346 | nn.GELU() |
| 347 | ) if not glu else GEGLU(dim, inner_dim) |
| 348 | |
| 349 | self.net = nn.Sequential( |
| 350 | project_in, |
| 351 | nn.Dropout(dropout), |
| 352 | nn.Linear(inner_dim, dim_out) |
| 353 | ) |
| 354 | |
| 355 | def forward(self, x): |
| 356 | return self.net(x) |
| 357 | |
| 358 | |
| 359 | class LinearAttention(nn.Module): |