| 264 | |
| 265 | |
| 266 | class BertSelfOutput(nn.Module): |
| 267 | def __init__(self, config): |
| 268 | super().__init__() |
| 269 | self.dense = nn.Linear(config.hidden_size, config.hidden_size) |
| 270 | self.LayerNorm = BertLayerNorm(config.hidden_size, eps=config.layer_norm_eps) |
| 271 | self.dropout = nn.Dropout(config.hidden_dropout_prob) |
| 272 | |
| 273 | def forward(self, hidden_states, input_tensor): |
| 274 | hidden_states = self.dense(hidden_states) |
| 275 | hidden_states = self.dropout(hidden_states) |
| 276 | hidden_states = self.LayerNorm(hidden_states + input_tensor) |
| 277 | return hidden_states |
| 278 | |
| 279 | |
| 280 | class BertAttention(nn.Module): |