Rotates half the hidden dims of the input.
(x)
| 196 | |
| 197 | |
| 198 | def rotate_half(x): |
| 199 | """Rotates half the hidden dims of the input.""" |
| 200 | x1 = x[..., : x.shape[-1] // 2] |
| 201 | x2 = x[..., x.shape[-1] // 2 :] |
| 202 | return torch.cat((-x2, x1), dim=-1) |
| 203 | |
| 204 | |
| 205 | def apply_rotary_pos_emb(q, k, cos, sin, position_ids=None, unsqueeze_dim=1): |
no outgoing calls
no test coverage detected