| 26 | return self.dropout(token_embedding + self.pos_embedding[:token_embedding.size(0), :]) |
| 27 | |
| 28 | class Translator(nn.Module): |
| 29 | def __init__( |
| 30 | self, |
| 31 | num_encoder_layers, |
| 32 | num_decoder_layers, |
| 33 | embed_size, |
| 34 | num_heads, |
| 35 | src_vocab_size, |
| 36 | tgt_vocab_size, |
| 37 | dim_feedforward, |
| 38 | dropout |
| 39 | ): |
| 40 | super(Translator, self).__init__() |
| 41 | |
| 42 | # Output of embedding must be equal (embed_size) |
| 43 | self.src_embedding = nn.Embedding(src_vocab_size, embed_size) |
| 44 | self.tgt_embedding = nn.Embedding(tgt_vocab_size, embed_size) |
| 45 | |
| 46 | self.pos_enc = PositionalEncoding(embed_size, dropout) |
| 47 | |
| 48 | self.transformer = nn.Transformer( |
| 49 | d_model=embed_size, |
| 50 | nhead=num_heads, |
| 51 | num_encoder_layers=num_encoder_layers, |
| 52 | num_decoder_layers=num_decoder_layers, |
| 53 | dim_feedforward=dim_feedforward, |
| 54 | dropout=dropout |
| 55 | ) |
| 56 | |
| 57 | self.ff = nn.Linear(embed_size, tgt_vocab_size) |
| 58 | |
| 59 | self._init_weights() |
| 60 | |
| 61 | def _init_weights(self): |
| 62 | for p in self.parameters(): |
| 63 | if p.dim() > 1: |
| 64 | nn.init.xavier_uniform_(p) |
| 65 | |
| 66 | def forward(self, src, trg, src_mask, tgt_mask, src_padding_mask, tgt_padding_mask, memory_key_padding_mask): |
| 67 | |
| 68 | src_emb = self.pos_enc(self.src_embedding(src)) |
| 69 | tgt_emb = self.pos_enc(self.tgt_embedding(trg)) |
| 70 | |
| 71 | outs = self.transformer( |
| 72 | src_emb, |
| 73 | tgt_emb, |
| 74 | src_mask, |
| 75 | tgt_mask, |
| 76 | None, |
| 77 | src_padding_mask, |
| 78 | tgt_padding_mask, |
| 79 | memory_key_padding_mask |
| 80 | ) |
| 81 | |
| 82 | return self.ff(outs) |
| 83 | |
| 84 | def encode(self, src, src_mask): |
| 85 | |