| 42 | |
| 43 | |
| 44 | class LayerNormalization(nn.Module): |
| 45 | def __init__(self, d_hid, eps=1e-6): |
| 46 | super(LayerNormalization, self).__init__() |
| 47 | self.gamma = nn.Parameter(torch.ones(d_hid)) |
| 48 | self.beta = nn.Parameter(torch.zeros(d_hid)) |
| 49 | self.eps = eps |
| 50 | |
| 51 | def forward(self, z): |
| 52 | mean = z.mean(dim=-1, keepdim=True,) |
| 53 | std = z.std(dim=-1, keepdim=True,) |
| 54 | ln_out = (z - mean) / (std + self.eps) |
| 55 | ln_out = self.gamma * ln_out + self.beta |
| 56 | |
| 57 | return ln_out |
| 58 | |
| 59 | |
| 60 | class PosEncoding(nn.Module): |