A residual connection followed by a layer norm. Note for code simplicity the norm is first as opposed to last.
| 280 | |
| 281 | # Adapted from The Annotated Transformer |
| 282 | class SublayerConnection(nn.Module): |
| 283 | """ |
| 284 | A residual connection followed by a layer norm. |
| 285 | Note for code simplicity the norm is first as opposed to last. |
| 286 | """ |
| 287 | def __init__(self, size, dropout): |
| 288 | super(SublayerConnection, self).__init__() |
| 289 | self.norm = nn.LayerNorm(size) |
| 290 | self.dropout = nn.Dropout(dropout) |
| 291 | |
| 292 | def forward(self, x, sublayer): |
| 293 | "Apply residual connection to any sublayer with the same size." |
| 294 | return x + self.dropout(sublayer(self.norm(x))) |
| 295 | |
| 296 | |
| 297 | # Adapted from The Annotated Transformer |