(x)
| 18 | |
| 19 | # rotary pos emb helpers (torch.jit.script does not seem to support staticmethod...) |
| 20 | def rotate_half(x): |
| 21 | x1, x2 = x[..., : x.shape[-1] // 2], x[..., x.shape[-1] // 2 :] |
| 22 | return torch.cat( |
| 23 | (-x2, x1), dim=x1.ndim - 1 |
| 24 | ) # dim=-1 triggers a bug in torch < 1.8.0 |
| 25 | |
| 26 | |
| 27 | class RotaryEmbedding(nn.Module): |