Function gelu_impl

codegeex/megatron/model/utils.py:62–66

OpenAI's gelu implementation.

(x)

@torch.jit.script
def gelu_impl(x):
    """OpenAI's gelu implementation."""
    return (
            0.5 * x * (1.0 + torch.tanh(0.7978845608028654 * x * (1.0 + 0.044715 * x * x)))
    )


def openai_gelu(x):