| 349 | |
| 350 | |
| 351 | class BertLayer(nn.Module): |
| 352 | def __init__(self, config): |
| 353 | super().__init__() |
| 354 | self.attention = BertAttention(config) |
| 355 | self.is_decoder = config.is_decoder |
| 356 | if self.is_decoder: |
| 357 | self.crossattention = BertAttention(config) |
| 358 | self.intermediate = BertIntermediate(config) |
| 359 | self.output = BertOutput(config) |
| 360 | |
| 361 | def forward( |
| 362 | self, |
| 363 | hidden_states, |
| 364 | attention_mask=None, |
| 365 | head_mask=None, |
| 366 | encoder_hidden_states=None, |
| 367 | encoder_attention_mask=None, |
| 368 | output_attentions=False, |
| 369 | ): |
| 370 | self_attention_outputs = self.attention( |
| 371 | hidden_states, attention_mask, head_mask, output_attentions=output_attentions, |
| 372 | ) |
| 373 | attention_output = self_attention_outputs[0] |
| 374 | outputs = self_attention_outputs[1:] # add self attentions if we output attention weights |
| 375 | |
| 376 | if self.is_decoder and encoder_hidden_states is not None: |
| 377 | cross_attention_outputs = self.crossattention( |
| 378 | attention_output, |
| 379 | attention_mask, |
| 380 | head_mask, |
| 381 | encoder_hidden_states, |
| 382 | encoder_attention_mask, |
| 383 | output_attentions, |
| 384 | ) |
| 385 | attention_output = cross_attention_outputs[0] |
| 386 | outputs = outputs + cross_attention_outputs[1:] # add cross attentions if we output attention weights |
| 387 | |
| 388 | intermediate_output = self.intermediate(attention_output) |
| 389 | layer_output = self.output(intermediate_output, attention_output) |
| 390 | outputs = (layer_output,) + outputs |
| 391 | return outputs |
| 392 | |
| 393 | |
| 394 | class BertEncoder(nn.Module): |