| 652 | }; |
| 653 | |
| 654 | tensor_offloading_levels get_offloading_level(llm_tensor tensor) { |
| 655 | switch (tensor) { |
| 656 | case LLM_TENSOR_TOKEN_EMBD: case LLM_TENSOR_TOKEN_EMBD_NORM: case LLM_TENSOR_POS_EMBD: |
| 657 | case LLM_TENSOR_ROPE_FREQS: |
| 658 | return TENSOR_NO_OFFLOAD; |
| 659 | case LLM_TENSOR_OUTPUT: case LLM_TENSOR_OUTPUT_NORM: |
| 660 | return TENSOR_OFFLOAD_OUTPUT; |
| 661 | case LLM_TENSOR_ATTN_Q: case LLM_TENSOR_ATTN_K: case LLM_TENSOR_ATTN_V: |
| 662 | case LLM_TENSOR_ATTN_QKV: case LLM_TENSOR_ATTN_OUT: case LLM_TENSOR_ATTN_NORM: |
| 663 | case LLM_TENSOR_ATTN_NORM_2: case LLM_TENSOR_ATTN_ROT_EMBD: |
| 664 | case LLM_TENSOR_ATTN_Q_NORM: case LLM_TENSOR_ATTN_K_NORM: |
| 665 | return TENSOR_OFFLOAD_ATTN; |
| 666 | case LLM_TENSOR_FFN_GATE: case LLM_TENSOR_FFN_DOWN: case LLM_TENSOR_FFN_UP: |
| 667 | case LLM_TENSOR_FFN_DOWN_T: |
| 668 | return TENSOR_OFFLOAD_FFN; |
| 669 | case LLM_TENSOR_FFN_NORM: |
| 670 | return TENSOR_OFFLOAD_FFN_IO; |
| 671 | case LLM_TENSOR_MLP_PRED_FC1: case LLM_TENSOR_MLP_PRED_FC2: |
| 672 | return TENSOR_OFFLOAD_MLP_PRED; |
| 673 | default: |
| 674 | throw std::runtime_error("unknown tensor category"); |
| 675 | } |
| 676 | } |
| 677 | |
| 678 | |
| 679 | // helper to handle gguf constants |