(self, config)
| 297 | ) |
| 298 | class CTRLModel(CTRLPreTrainedModel): |
| 299 | def __init__(self, config): |
| 300 | super().__init__(config) |
| 301 | |
| 302 | self.d_model_size = config.n_embd |
| 303 | self.num_layers = config.n_layer |
| 304 | |
| 305 | self.pos_encoding = positional_encoding(config.n_positions, self.d_model_size, torch.float) |
| 306 | |
| 307 | self.w = nn.Embedding(config.vocab_size, config.n_embd) |
| 308 | |
| 309 | self.dropout = nn.Dropout(config.embd_pdrop) |
| 310 | self.h = nn.ModuleList( |
| 311 | [EncoderLayer(config.n_embd, config.n_head, config.dff, config.resid_pdrop) for _ in range(config.n_layer)] |
| 312 | ) |
| 313 | self.layernorm = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon) |
| 314 | |
| 315 | self.init_weights() |
| 316 | |
| 317 | def get_input_embeddings(self): |
| 318 | return self.w |
no test coverage detected