Initialize the weights.
(self, module: nn.Module)
| 573 | super().__init__(*inputs, **kwargs) |
| 574 | |
| 575 | def _init_weights(self, module: nn.Module): |
| 576 | """Initialize the weights.""" |
| 577 | if isinstance(module, nn.Linear): |
| 578 | module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) |
| 579 | if module.bias is not None: |
| 580 | module.bias.data.zero_() |
| 581 | |
| 582 | elif isinstance(module, nn.Embedding): |
| 583 | module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) |
| 584 | if module.padding_idx is not None: |
| 585 | module.weight.data[module.padding_idx].zero_() |
| 586 | |
| 587 | elif isinstance(module, LayerNorm): |
| 588 | module.bias.data.zero_() |
| 589 | module.weight.data.fill_(1.0) |
| 590 | |
| 591 | def _set_gradient_checkpointing(self, module: nn.Module, value: bool = False): |
| 592 | if isinstance(module, TelechatModel): |
nothing calls this directly
no outgoing calls
no test coverage detected