Init method based on N(0, sigma/sqrt(2*num_layers).
(mean, std, num_layers)
| 47 | |
| 48 | |
| 49 | def scaled_init_method(mean, std, num_layers): |
| 50 | """Init method based on N(0, sigma/sqrt(2*num_layers).""" |
| 51 | std = std / math.sqrt(2.0 * num_layers) |
| 52 | |
| 53 | def init_(tensor): |
| 54 | return torch.nn.init.normal_(tensor, mean=mean, std=std) |
| 55 | |
| 56 | return init_ |
| 57 | |
| 58 | |
| 59 | def bert_extended_attention_mask(attention_mask): |