(dim, position)
| 26 | |
| 27 | |
| 28 | def sinusoidal_embedding_1d(dim, position): |
| 29 | # preprocess |
| 30 | assert dim % 2 == 0 |
| 31 | half = dim // 2 |
| 32 | position = position.type(torch.float64) |
| 33 | |
| 34 | # calculation |
| 35 | sinusoid = torch.outer( |
| 36 | position, torch.pow(10000, -torch.arange(half).to(position).div(half))) |
| 37 | x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) |
| 38 | return x |
| 39 | |
| 40 | |
| 41 | @amp.autocast(enabled=False) |