Init method based on N(0, sigma/sqrt(2*num_layers).
(sigma, num_layers)
| 29 | |
| 30 | |
| 31 | def scaled_init_method_normal(sigma, num_layers): |
| 32 | """Init method based on N(0, sigma/sqrt(2*num_layers).""" |
| 33 | std = sigma / math.sqrt(2.0 * num_layers) |
| 34 | |
| 35 | def init_(tensor): |
| 36 | return torch.nn.init.normal_(tensor, mean=0.0, std=std) |
| 37 | |
| 38 | return init_ |
| 39 | |
| 40 | |
| 41 | def attention_mask_func(attention_scores, attention_mask): |
no outgoing calls
no test coverage detected