(n_channels, dim)
| 78 | # positional |
| 79 | |
| 80 | def sinusoidal_embedding(n_channels, dim): |
| 81 | pe = torch.FloatTensor([[p / (10000 ** (2 * (i // 2) / dim)) for i in range(dim)] |
| 82 | for p in range(n_channels)]) |
| 83 | pe[:, 0::2] = torch.sin(pe[:, 0::2]) |
| 84 | pe[:, 1::2] = torch.cos(pe[:, 1::2]) |
| 85 | return rearrange(pe, '... -> 1 ...') |
| 86 | |
| 87 | |
| 88 | # modules |