(
self,
d_model,
nhead,
dim_feedforward=2048,
attention_dropout_rate=0.0,
residual_dropout_rate=0.1,
with_self_attn=True,
with_cross_attn=False,
epsilon=1e-5,
)
| 522 | class TransformerBlock(nn.Module): |
| 523 | |
| 524 | def __init__( |
| 525 | self, |
| 526 | d_model, |
| 527 | nhead, |
| 528 | dim_feedforward=2048, |
| 529 | attention_dropout_rate=0.0, |
| 530 | residual_dropout_rate=0.1, |
| 531 | with_self_attn=True, |
| 532 | with_cross_attn=False, |
| 533 | epsilon=1e-5, |
| 534 | ): |
| 535 | super(TransformerBlock, self).__init__() |
| 536 | self.with_self_attn = with_self_attn |
| 537 | if with_self_attn: |
| 538 | self.self_attn = MultiheadAttention(d_model, |
| 539 | nhead, |
| 540 | dropout=attention_dropout_rate, |
| 541 | self_attn=with_self_attn) |
| 542 | self.norm1 = nn.LayerNorm(d_model, eps=epsilon) |
| 543 | self.dropout1 = nn.Dropout(residual_dropout_rate) |
| 544 | self.with_cross_attn = with_cross_attn |
| 545 | if with_cross_attn: |
| 546 | self.cross_attn = MultiheadAttention( |
| 547 | d_model, nhead, dropout=attention_dropout_rate |
| 548 | ) # for self_attn of encoder or cross_attn of decoder |
| 549 | self.norm2 = nn.LayerNorm(d_model, eps=epsilon) |
| 550 | self.dropout2 = nn.Dropout(residual_dropout_rate) |
| 551 | |
| 552 | self.mlp = Mlp( |
| 553 | in_features=d_model, |
| 554 | hidden_features=dim_feedforward, |
| 555 | act_layer=nn.ReLU, |
| 556 | drop=residual_dropout_rate, |
| 557 | ) |
| 558 | |
| 559 | self.norm3 = nn.LayerNorm(d_model, eps=epsilon) |
| 560 | |
| 561 | self.dropout3 = nn.Dropout(residual_dropout_rate) |
| 562 | |
| 563 | def forward(self, |
| 564 | tgt, |
nothing calls this directly
no test coverage detected