| 95 | |
| 96 | |
| 97 | class EncoderLayer(nn.Module): |
| 98 | def __init__(self, d_model, heads, dropout=0.1): |
| 99 | super().__init__() |
| 100 | self.norm_1 = Norm(d_model) |
| 101 | self.norm_2 = Norm(d_model) |
| 102 | self.attn = MultiHeadAttention(heads, d_model, dropout=dropout) |
| 103 | self.ff = FeedForward(d_model, dropout=dropout) |
| 104 | self.dropout_1 = nn.Dropout(dropout) |
| 105 | self.dropout_2 = nn.Dropout(dropout) |
| 106 | |
| 107 | def forward(self, x, mask): |
| 108 | x2 = self.norm_1(x) |
| 109 | x = x + self.dropout_1(self.attn(x2, x2, x2, mask)) |
| 110 | x2 = self.norm_2(x) |
| 111 | x = x + self.dropout_2(self.ff(x2)) |
| 112 | return x |
| 113 | |
| 114 | |
| 115 |
nothing calls this directly
no outgoing calls
no test coverage detected