| 3 | import torch.nn.functional as F |
| 4 | |
| 5 | class GELU(nn.Module): |
| 6 | def __init__(self): |
| 7 | super().__init__() |
| 8 | |
| 9 | def forward(self, x): |
| 10 | return 0.5 * x * (1.0 + torch.tanh(0.7978845608 * (x + 0.044715 * torch.pow(x, 3)))) |
| 11 | |
| 12 | class Swish(nn.Module): |
| 13 | def __init__(self): |