| 432 | |
| 433 | |
| 434 | class BertOutput(nn.Module): |
| 435 | def __init__(self, config): |
| 436 | super(BertOutput, self).__init__() |
| 437 | self.dense = nn.Linear(config.intermediate_size, config.hidden_size) |
| 438 | self.LayerNorm = BertLayerNorm(config.hidden_size, eps=1e-12) |
| 439 | self.dropout = nn.Dropout(config.hidden_dropout_prob) |
| 440 | |
| 441 | def forward(self, hidden_states, input_tensor): |
| 442 | hidden_states = self.dense(hidden_states) |
| 443 | hidden_states = self.dropout(hidden_states) |
| 444 | hidden_states = self.LayerNorm(hidden_states + input_tensor) |
| 445 | return hidden_states |
| 446 | |
| 447 | |
| 448 | class BertLayer(nn.Module): |