| 138 | super().__init__(*args, **kwargs, bias=False) |
| 139 | |
| 140 | class Norm(nn.Module): |
| 141 | def __init__(self, |
| 142 | dim: int, |
| 143 | eps: float = 1e-5,) -> None: |
| 144 | super().__init__() |
| 145 | self.eps = eps |
| 146 | self.weight = nn.Parameter(T.ones((dim,))) |
| 147 | |
| 148 | def forward(self, input: Tensor) -> Tensor: |
| 149 | return F.layer_norm(input, (self.weight.shape[0],), weight=self.weight, bias=None, eps=self.eps) |
| 150 | |
| 151 | |
| 152 | class FFNN(nn.Module): |