| 86 | |
| 87 | |
| 88 | class Positional_Encoding(nn.Module): |
| 89 | def __init__(self, embed, pad_size, dropout, device): |
| 90 | super(Positional_Encoding, self).__init__() |
| 91 | self.device = device |
| 92 | self.pe = torch.tensor([[pos / (10000.0 ** (i // 2 * 2.0 / embed)) for i in range(embed)] for pos in range(pad_size)]) |
| 93 | self.pe[:, 0::2] = np.sin(self.pe[:, 0::2]) |
| 94 | self.pe[:, 1::2] = np.cos(self.pe[:, 1::2]) |
| 95 | self.dropout = nn.Dropout(dropout) |
| 96 | |
| 97 | def forward(self, x): |
| 98 | out = x + nn.Parameter(self.pe, requires_grad=False).to(self.device) |
| 99 | out = self.dropout(out) |
| 100 | return out |
| 101 | |
| 102 | |
| 103 | class Scaled_Dot_Product_Attention(nn.Module): |