(self)
| 427 | trunc_normal_(self.pos_embed, std=.02) |
| 428 | |
| 429 | def fix_init_weight(self): |
| 430 | def rescale(param, layer_id): |
| 431 | param.div_(math.sqrt(2.0 * layer_id)) |
| 432 | |
| 433 | for layer_id, layer in enumerate(self.blocks): |
| 434 | rescale(layer.attn.proj.weight.data, layer_id + 1) |
| 435 | rescale(layer.mlp.fc2.weight.data, layer_id + 1) |
| 436 | |
| 437 | def _init_weights(self, m): |
| 438 | if isinstance(m, nn.Linear): |