| 422 | |
| 423 | |
| 424 | class BertIntermediate(nn.Module): |
| 425 | def __init__(self, config): |
| 426 | super(BertIntermediate, self).__init__() |
| 427 | self.dense_act = LinearActivation(config.hidden_size, config.intermediate_size, act=config.hidden_act) |
| 428 | |
| 429 | def forward(self, hidden_states): |
| 430 | hidden_states = self.dense_act(hidden_states) |
| 431 | return hidden_states |
| 432 | |
| 433 | |
| 434 | class BertOutput(nn.Module): |