Base class for VTP models.
| 29 | |
| 30 | |
| 31 | class VTPPreTrainedModel(PreTrainedModel): |
| 32 | """Base class for VTP models.""" |
| 33 | |
| 34 | config_class = VTPConfig |
| 35 | base_model_prefix = "vtp" |
| 36 | supports_gradient_checkpointing = True |
| 37 | |
| 38 | def _init_weights(self, module): |
| 39 | """Initialize weights following VTP conventions.""" |
| 40 | if isinstance(module, nn.Linear): |
| 41 | nn.init.trunc_normal_(module.weight, std=0.02) |
| 42 | if module.bias is not None: |
| 43 | nn.init.zeros_(module.bias) |
| 44 | elif isinstance(module, nn.LayerNorm): |
| 45 | nn.init.ones_(module.weight) |
| 46 | nn.init.zeros_(module.bias) |
| 47 | elif isinstance(module, nn.Embedding): |
| 48 | nn.init.normal_(module.weight, std=0.02) |
| 49 | |
| 50 | |
| 51 | class VTPModel(VTPPreTrainedModel): |
nothing calls this directly
no outgoing calls
no test coverage detected