Class GELU_

model/transformer_utils.py:58–63

Fast gelu implementation.

class GELU_(nn.Module):
    """
    Fast gelu implementation.
    """
    def forward(self, x):
        return x * torch.sigmoid(1.702 * x)
    
class LayerNorm(nn.RMSNorm):
    """