| 487 | """ |
| 488 | |
| 489 | def __init__(self, dim, eps: float = 1e-5, elementwise_affine: bool = True, bias: bool = True): |
| 490 | super().__init__() |
| 491 | |
| 492 | self.eps = eps |
| 493 | |
| 494 | if isinstance(dim, numbers.Integral): |
| 495 | dim = (dim,) |
| 496 | |
| 497 | self.dim = torch.Size(dim) |
| 498 | |
| 499 | if elementwise_affine: |
| 500 | self.weight = nn.Parameter(torch.ones(dim)) |
| 501 | self.bias = nn.Parameter(torch.zeros(dim)) if bias else None |
| 502 | else: |
| 503 | self.weight = None |
| 504 | self.bias = None |
| 505 | |
| 506 | def forward(self, input): |
| 507 | return F.layer_norm(input, self.dim, self.weight, self.bias, self.eps) |