| 6 | gelu = nn.functional.gelu |
| 7 | |
| 8 | class lm_head(torch.nn.Module): |
| 9 | def __init__(self, vocab_size, hidden_size, layernorm_epsilon=1.0e-5): |
| 10 | super().__init__() |
| 11 | self.dense = nn.Linear(hidden_size, hidden_size) |
| 12 | self.layer_norm = LayerNorm(hidden_size, eps=layernorm_epsilon) |
| 13 | self.decoder = nn.Linear(hidden_size, vocab_size) |
| 14 | |
| 15 | def forward(self, x): |
| 16 | x = self.dense(x) |
| 17 | x = gelu(x) |
| 18 | x = self.layer_norm(x) |
| 19 | x = self.decoder(x) |
| 20 | return x |
| 21 | |
| 22 | class BertFinalMixin(BaseMixin): |
| 23 | def __init__(self, vocab_size, hidden_size): |
no outgoing calls
no test coverage detected