(self, hidden_size)
| 37 | |
| 38 | class PositionalEmbedding(torch.nn.Module): |
| 39 | def __init__(self, hidden_size): |
| 40 | super(PositionalEmbedding, self).__init__() |
| 41 | |
| 42 | self.hidden_size = hidden_size |
| 43 | |
| 44 | inv_freq = 1 / (10000 ** (torch.arange(0.0, hidden_size, 2.0) / hidden_size)) |
| 45 | self.register_buffer('inv_freq', inv_freq) |
| 46 | |
| 47 | def forward(self, pos_seq, bsz=None): |
| 48 | sinusoid_inp = torch.ger(pos_seq, self.inv_freq) |