| 320 | |
| 321 | |
| 322 | class BertIntermediate(nn.Module): |
| 323 | def __init__(self, config): |
| 324 | super().__init__() |
| 325 | self.dense = nn.Linear(config.hidden_size, config.intermediate_size) |
| 326 | if isinstance(config.hidden_act, str): |
| 327 | self.intermediate_act_fn = ACT2FN[config.hidden_act] |
| 328 | else: |
| 329 | self.intermediate_act_fn = config.hidden_act |
| 330 | |
| 331 | def forward(self, hidden_states): |
| 332 | hidden_states = self.dense(hidden_states) |
| 333 | hidden_states = self.intermediate_act_fn(hidden_states) |
| 334 | return hidden_states |
| 335 | |
| 336 | |
| 337 | class BertOutput(nn.Module): |