Layer normalization over last dimension.
(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float = 1e-5)
| 18 | |
| 19 | # Layer Normalization |
| 20 | def layernorm_ref(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor, eps: float = 1e-5) -> torch.Tensor: |
| 21 | """Layer normalization over last dimension.""" |
| 22 | normalized_shape = x.shape[-1:] |
| 23 | return F.layer_norm(x, normalized_shape, weight, bias, eps) |
| 24 | |
| 25 | # RMS Normalization |
| 26 | def rmsnorm_ref(x: torch.Tensor, weight: torch.Tensor, eps: float = 1e-6) -> torch.Tensor: |
nothing calls this directly
no outgoing calls
no test coverage detected