(max_seq_len, dim, theta=10000)
| 40 | |
| 41 | @amp.autocast(enabled=False) |
| 42 | def rope_params(max_seq_len, dim, theta=10000): |
| 43 | |
| 44 | assert dim % 2 == 0 |
| 45 | freqs = torch.outer( |
| 46 | torch.arange(max_seq_len), |
| 47 | 1.0 / torch.pow(theta, |
| 48 | torch.arange(0, dim, 2).to(torch.float64).div(dim))) |
| 49 | freqs = torch.polar(torch.ones_like(freqs), freqs) |
| 50 | return freqs |
| 51 | |
| 52 | |
| 53 | @amp.autocast(enabled=False) |
no outgoing calls
no test coverage detected