| 83 | return self.dropout(x) |
| 84 | |
| 85 | class LearnablePositionalEncoding(nn.Module): |
| 86 | |
| 87 | def __init__(self, d_model, dropout=0.1, max_len=1024): |
| 88 | super(LearnablePositionalEncoding, self).__init__() |
| 89 | self.dropout = nn.Dropout(p=dropout) |
| 90 | # Each position gets its own embedding |
| 91 | # Since indices are always 0 ... max_len, we don't have to do a look-up |
| 92 | self.pe = nn.Parameter(torch.empty(max_len, d_model)) # requires_grad automatically set to True |
| 93 | nn.init.uniform_(self.pe, -0.02, 0.02) |
| 94 | |
| 95 | # distance = torch.matmul(self.pe, self.pe[10]) |
| 96 | # import matplotlib.pyplot as plt |
| 97 | |
| 98 | # plt.plot(distance.detach().numpy()) |
| 99 | # plt.show() |
| 100 | |
| 101 | def forward(self, x): |
| 102 | r"""Inputs of forward function |
| 103 | Args: |
| 104 | x: the sequence fed to the positional encoder model (required). |
| 105 | Shape: |
| 106 | x: [sequence length, batch size, embed dim] |
| 107 | output: [sequence length, batch size, embed dim] |
| 108 | """ |
| 109 | |
| 110 | x = x + self.pe |
| 111 | # distance = torch.matmul(self.pe, self.pe.transpose(1,0)) |
| 112 | # distance_pd = pd.DataFrame(distance.cpu().detach().numpy()) |
| 113 | # distance_pd.to_csv('learn_position_distance.csv') |
| 114 | return self.dropout(x) |