Init method based on N(0, sigma).
(sigma)
| 82 | return int(math.sqrt(x) + 1e-4) |
| 83 | |
| 84 | def unscaled_init_method(sigma): |
| 85 | """Init method based on N(0, sigma).""" |
| 86 | def init_(tensor, **kwargs): |
| 87 | return torch.nn.init.normal_(tensor, mean=0.0, std=sigma) |
| 88 | |
| 89 | return init_ |
| 90 | |
| 91 | def scaled_init_method(sigma, num_layers): |
| 92 | """Init method based on N(0, sigma/sqrt(2*num_layers).""" |