| 58 | |
| 59 | class InstantLayerNorm1d(nn.Module): |
| 60 | def __init__(self, |
| 61 | num_features, |
| 62 | affine=True, |
| 63 | eps=1e-5, |
| 64 | ): |
| 65 | super(InstantLayerNorm1d, self).__init__() |
| 66 | self.num_features = num_features |
| 67 | self.affine = affine |
| 68 | self.eps = eps |
| 69 | |
| 70 | if affine: |
| 71 | self.gain = nn.Parameter(torch.ones(1, 1, num_features), requires_grad=True) |
| 72 | self.bias = nn.Parameter(torch.zeros(1, 1, num_features), requires_grad=True) |
| 73 | else: |
| 74 | self.gain = Variable(torch.ones(1, 1, num_features), requires_grad=False) |
| 75 | self.bias = Variable(torch.zeros(1, 1, num_features), requires_gra=False) |
| 76 | |
| 77 | def forward(self, inpt): |
| 78 | # inpt: (T,B,C) |