A two-feed-forward-layer module
| 27 | |
| 28 | |
| 29 | class PositionwiseFeedForward(nn.Module): |
| 30 | ''' A two-feed-forward-layer module ''' |
| 31 | |
| 32 | def __init__(self, d_in, d_hid, dropout=0.1): |
| 33 | super(PositionwiseFeedForward, self).__init__() |
| 34 | self.w_1 = nn.Conv1d(d_in, d_hid, 1) # position-wise |
| 35 | self.w_2 = nn.Conv1d(d_hid, d_in, 1) # position-wise |
| 36 | self.layer_norm = nn.LayerNorm(d_in) |
| 37 | self.dropout = nn.Dropout(dropout) |
| 38 | |
| 39 | def forward(self, x): |
| 40 | residual = x |
| 41 | output = x.transpose(1, 2) |
| 42 | output = self.w_2(F.relu(self.w_1(output))) |
| 43 | output = output.transpose(1, 2) |
| 44 | output = self.dropout(output) |
| 45 | output = self.layer_norm(output + residual) |
| 46 | return output |
| 47 | |
| 48 | |
| 49 | class EncoderLayer(nn.Module): |