(x: torch.Tensor)
| 45 | |
| 46 | # Copied from transformers.models.gptj.modeling_gptj.rotate_every_two |
| 47 | def rotate_every_two(x: torch.Tensor) -> torch.Tensor: |
| 48 | x1 = x[:, :, :, ::2] |
| 49 | x2 = x[:, :, :, 1::2] |
| 50 | x = torch.stack((-x2, x1), dim=-1) |
| 51 | return x.flatten(-2) # in einsum notation: rearrange(x, '... d j -> ... (d j)') |
| 52 | |
| 53 | |
| 54 | # Copied from transformers.models.gptj.modeling_gptj.apply_rotary_pos_emb |
no outgoing calls
no test coverage detected