(dim, position)
| 208 | |
| 209 | |
| 210 | def sinusoidal_embedding_1d(dim, position): |
| 211 | # preprocess |
| 212 | assert dim % 2 == 0 |
| 213 | half = dim // 2 |
| 214 | position = position.type(torch.float64) |
| 215 | |
| 216 | # calculation |
| 217 | sinusoid = torch.outer( |
| 218 | position, torch.pow(10000, -torch.arange(half).to(position).div(half))) |
| 219 | x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) |
| 220 | return x |
| 221 | |
| 222 | |
| 223 | @amp.autocast(enabled=False, device_type="cuda") |