Construct a layernorm module in the TF style (epsilon inside the square root).
(self, hidden_size, eps=1e-12)
| 294 | #print("Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.") |
| 295 | class BertLayerNorm(nn.Module): |
| 296 | def __init__(self, hidden_size, eps=1e-12): |
| 297 | """Construct a layernorm module in the TF style (epsilon inside the square root). |
| 298 | """ |
| 299 | super(BertLayerNorm, self).__init__() |
| 300 | self.weight = nn.Parameter(torch.ones(hidden_size)) |
| 301 | self.bias = nn.Parameter(torch.zeros(hidden_size)) |
| 302 | self.variance_epsilon = eps |
| 303 | |
| 304 | def forward(self, x): |
| 305 | u = x.mean(-1, keepdim=True) |