An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models.
| 621 | |
| 622 | |
| 623 | class BertPreTrainedModel(PreTrainedModel): |
| 624 | """ |
| 625 | An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained |
| 626 | models. |
| 627 | """ |
| 628 | |
| 629 | config_class = BertConfig |
| 630 | base_model_prefix = "bert" |
| 631 | _keys_to_ignore_on_load_missing = [r"position_ids"] |
| 632 | |
| 633 | def _init_weights(self, module): |
| 634 | """ Initialize the weights """ |
| 635 | if isinstance(module, (nn.Linear, nn.Embedding)): |
| 636 | # Slightly different from the TF version which uses truncated_normal for initialization |
| 637 | # cf https://github.com/pytorch/pytorch/pull/5617 |
| 638 | module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) |
| 639 | elif isinstance(module, nn.LayerNorm): |
| 640 | module.bias.data.zero_() |
| 641 | module.weight.data.fill_(1.0) |
| 642 | if isinstance(module, nn.Linear) and module.bias is not None: |
| 643 | module.bias.data.zero_() |
| 644 | |
| 645 | |
| 646 | class BertModel(BertPreTrainedModel): |
nothing calls this directly
no outgoing calls
no test coverage detected