| 37 | |
| 38 | |
| 39 | class FixedPositionalEmbedding(nn.Module): |
| 40 | def __init__(self, dim): |
| 41 | super().__init__() |
| 42 | inv_freq = 1. / (10000 ** (torch.arange(0, dim, 2).float() / dim)) |
| 43 | self.register_buffer('inv_freq', inv_freq) |
| 44 | |
| 45 | def forward(self, x, seq_dim=1, offset=0): |
| 46 | t = torch.arange(x.shape[seq_dim], device=x.device).type_as(self.inv_freq) + offset |
| 47 | sinusoid_inp = torch.einsum('i , j -> i j', t, self.inv_freq) |
| 48 | emb = torch.cat((sinusoid_inp.sin(), sinusoid_inp.cos()), dim=-1) |
| 49 | return emb[None, :, :] |
| 50 | |
| 51 | |
| 52 | # helpers |