Initialize the RMSNorm normalization layer. Args: dim (int): The dimension of the input tensor. eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. Attributes: eps (float): A small valu
(
self,
dim: int,
elementwise_affine=True,
eps: float = 1e-6,
device=None,
dtype=None,
)
| 19 | |
| 20 | class RMSNorm(nn.Module): |
| 21 | def __init__( |
| 22 | self, |
| 23 | dim: int, |
| 24 | elementwise_affine=True, |
| 25 | eps: float = 1e-6, |
| 26 | device=None, |
| 27 | dtype=None, |
| 28 | ): |
| 29 | """ |
| 30 | Initialize the RMSNorm normalization layer. |
| 31 | |
| 32 | Args: |
| 33 | dim (int): The dimension of the input tensor. |
| 34 | eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. |
| 35 | |
| 36 | Attributes: |
| 37 | eps (float): A small value added to the denominator for numerical stability. |
| 38 | weight (nn.Parameter): Learnable scaling parameter. |
| 39 | |
| 40 | """ |
| 41 | factory_kwargs = {"device": device, "dtype": dtype} |
| 42 | super().__init__() |
| 43 | self.eps = eps |
| 44 | if elementwise_affine: |
| 45 | self.weight = nn.Parameter(torch.ones(dim, **factory_kwargs)) |
| 46 | |
| 47 | def _norm(self, x): |
| 48 | """ |