| 569 | self.bias = bias |
| 570 | |
| 571 | def initialize(self, x): |
| 572 | self.in_features = x.shape[-1] |
| 573 | w_shape = (self.in_features, self.out_features) |
| 574 | b_shape = (self.out_features,) |
| 575 | |
| 576 | self.W = Tensor(shape=w_shape, |
| 577 | dtype=x.dtype, |
| 578 | requires_grad=True, |
| 579 | stores_grad=True) |
| 580 | std = math.sqrt(2.0 / (self.in_features + self.out_features)) |
| 581 | self.W.gaussian(0.0, std) |
| 582 | |
| 583 | if self.bias: |
| 584 | self.b = Tensor(shape=b_shape, |
| 585 | dtype=x.dtype, |
| 586 | requires_grad=True, |
| 587 | stores_grad=True) |
| 588 | self.b.set_value(0.0) |
| 589 | else: |
| 590 | self.b = None |
| 591 | |
| 592 | def forward(self, x): |
| 593 | if self.b: |