implementation of the 2D RoPE without adding a new op in ggml this is not efficient (use double the memory), but works on all backends TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pu
| 644 | // this is not efficient (use double the memory), but works on all backends |
| 645 | // TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065 |
| 646 | ggml_tensor * clip_graph::build_rope_2d( |
| 647 | ggml_context * ctx0, |
| 648 | ggml_tensor * cur, |
| 649 | ggml_tensor * pos_a, // first half |
| 650 | ggml_tensor * pos_b, // second half |
| 651 | const float freq_base, |
| 652 | const bool interleave_freq |
| 653 | ) { |
| 654 | const int64_t n_dim = cur->ne[0]; |
| 655 | const int64_t n_head = cur->ne[1]; |
| 656 | const int64_t n_pos = cur->ne[2]; |
| 657 | |
| 658 | // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos) |
| 659 | // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3 |
| 660 | // first half of cur will use 1e-0, 1e-2 (even) |
| 661 | // second half of cur will use 1e-1, 1e-3 (odd) |
| 662 | // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even |
| 663 | // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2) |
| 664 | // then for the second half, we use freq_scale to shift the inv_freq |
| 665 | // ^ why? replace (2i) with (2i+1) in the above equation |
| 666 | const float freq_scale_odd = interleave_freq |
| 667 | ? std::pow(freq_base, (float)-2/n_dim) |
| 668 | : 1.0; |
| 669 | |
| 670 | // first half |
| 671 | ggml_tensor * first; |
| 672 | { |
| 673 | first = ggml_view_3d(ctx0, cur, |
| 674 | n_dim/2, n_head, n_pos, |
| 675 | ggml_row_size(cur->type, n_dim), |
| 676 | ggml_row_size(cur->type, n_dim*n_head), |
| 677 | 0); |
| 678 | first = ggml_rope_ext( |
| 679 | ctx0, |
| 680 | first, |
| 681 | pos_a, // positions |
| 682 | nullptr, // freq factors |
| 683 | n_dim/2, // n_dims |
| 684 | 0, 0, freq_base, |
| 685 | 1.0f, 0.0f, 1.0f, 0.0f, 0.0f |
| 686 | ); |
| 687 | } |
| 688 | |
| 689 | // second half |
| 690 | ggml_tensor * second; |
| 691 | { |
| 692 | second = ggml_view_3d(ctx0, cur, |
| 693 | n_dim/2, n_head, n_pos, |
| 694 | ggml_row_size(cur->type, n_dim), |
| 695 | ggml_row_size(cur->type, n_dim*n_head), |
| 696 | n_dim/2 * ggml_element_size(cur)); |
| 697 | second = ggml_rope_ext( |
| 698 | ctx0, |
| 699 | second, |
| 700 | pos_b, // positions |
| 701 | nullptr, // freq factors |
| 702 | n_dim/2, // n_dims |
| 703 | 0, 0, freq_base, |
nothing calls this directly
no test coverage detected