| 515 | |
| 516 | |
| 517 | class LayerNorm(layer.Layer): |
| 518 | def __init__(self, n_features, eps=1e-6): |
| 519 | super(LayerNorm, self).__init__() |
| 520 | self.n_features = n_features |
| 521 | self.eps = eps |
| 522 | |
| 523 | def initialize(self, x): |
| 524 | shape = (self.n_features,) |
| 525 | self.Gamma = Tensor(shape=shape, dtype=x.dtype, requires_grad=False, stores_grad=False) |
| 526 | self.Beta = Tensor(shape=shape, dtype=x.dtype, requires_grad=False, stores_grad=False) |
| 527 | self.Gamma.set_value(1.0) |
| 528 | self.Beta.set_value(0.0) |
| 529 | |
| 530 | def forward(self, x): |
| 531 | # x: input tensor with shape [batch_size, n_features] |
| 532 | # x_normalized = (x - tensor.from_numpy(self.mean)) / tensor.from_numpy(np.sqrt(self.var + self.eps)) |
| 533 | # y = self.gamma * x_normalized + self.beta |
| 534 | mean = np.mean(tensor.to_numpy(x), axis=-1, keepdims=True) |
| 535 | var = np.var(tensor.to_numpy(x), axis=-1, keepdims=True) |
| 536 | |
| 537 | sub1 = tensor.from_numpy(mean) |
| 538 | div1 = tensor.from_numpy(np.sqrt(var + self.eps)) |
| 539 | x_normalized = autograd.div(autograd.sub(x, sub1), div1) |
| 540 | y = autograd.mul(self.Gamma, x_normalized) |
| 541 | y = autograd.add(y, self.Beta) |
| 542 | return y |
| 543 | |
| 544 | |
| 545 | class Linear3D(layer.Layer): |