RMSNorm is equivalent to T5LayerNorm
(self, hidden_size, eps=1e-6)
| 27 | import torch.nn as nn |
| 28 | class RMSNorm(nn.Module): |
| 29 | def __init__(self, hidden_size, eps=1e-6): |
| 30 | """ |
| 31 | RMSNorm is equivalent to T5LayerNorm |
| 32 | """ |
| 33 | super().__init__() |
| 34 | self.weight = nn.Parameter(torch.ones(hidden_size)) |
| 35 | self.variance_epsilon = eps |
| 36 | |
| 37 | def forward(self, hidden_states): |
| 38 | input_dtype = hidden_states.dtype |