| 59 | |
| 60 | class PosEncoding(nn.Module): |
| 61 | def __init__(self, max_seq_len, d_word_vec): |
| 62 | super(PosEncoding, self).__init__() |
| 63 | pos_enc = np.array( |
| 64 | [[pos / np.power(10000, 2.0 * (j // 2) / d_word_vec) for j in range(d_word_vec)] |
| 65 | for pos in range(max_seq_len)]) |
| 66 | pos_enc[:, 0::2] = np.sin(pos_enc[:, 0::2]) |
| 67 | pos_enc[:, 1::2] = np.cos(pos_enc[:, 1::2]) |
| 68 | pad_row = np.zeros([1, d_word_vec]) |
| 69 | pos_enc = np.concatenate([pad_row, pos_enc]).astype(np.float32) |
| 70 | |
| 71 | # additional single row for PAD idx |
| 72 | self.pos_enc = nn.Embedding(max_seq_len + 1, d_word_vec) |
| 73 | # fix positional encoding: exclude weight from grad computation |
| 74 | self.pos_enc.weight = nn.Parameter(torch.from_numpy(pos_enc), requires_grad=False) |
| 75 | self.max_len = int(max_seq_len/10) |
| 76 | def forward(self, input_len): |
| 77 | max_len = self.max_len # torch.max(input_len) |
| 78 | tensor = torch.cuda.LongTensor if input_len.is_cuda else torch.LongTensor |