| 912 | } |
| 913 | |
| 914 | ggml_tensor * llm_graph_context::build_ffn( |
| 915 | ggml_tensor * cur, |
| 916 | ggml_tensor * up, |
| 917 | ggml_tensor * up_b, |
| 918 | ggml_tensor * up_s, |
| 919 | ggml_tensor * gate, |
| 920 | ggml_tensor * gate_b, |
| 921 | ggml_tensor * gate_s, |
| 922 | ggml_tensor * down, |
| 923 | ggml_tensor * down_b, |
| 924 | ggml_tensor * down_s, |
| 925 | ggml_tensor * act_scales, |
| 926 | llm_ffn_op_type type_op, |
| 927 | llm_ffn_gate_type type_gate, |
| 928 | int il) const { |
| 929 | ggml_tensor * tmp = up ? build_lora_mm(up, cur) : cur; |
| 930 | cb(tmp, "ffn_up", il); |
| 931 | |
| 932 | if (up_b) { |
| 933 | tmp = ggml_add(ctx0, tmp, up_b); |
| 934 | cb(tmp, "ffn_up_b", il); |
| 935 | } |
| 936 | |
| 937 | if (up_s) { |
| 938 | tmp = ggml_mul(ctx0, tmp, up_s); |
| 939 | cb(tmp, "ffn_up_s", il); |
| 940 | } |
| 941 | |
| 942 | if (gate) { |
| 943 | switch (type_gate) { |
| 944 | case LLM_FFN_SEQ: |
| 945 | { |
| 946 | cur = build_lora_mm(gate, tmp); |
| 947 | cb(cur, "ffn_gate", il); |
| 948 | } break; |
| 949 | case LLM_FFN_PAR: |
| 950 | { |
| 951 | cur = build_lora_mm(gate, cur); |
| 952 | cb(cur, "ffn_gate", il); |
| 953 | } break; |
| 954 | } |
| 955 | |
| 956 | if (gate_b) { |
| 957 | cur = ggml_add(ctx0, cur, gate_b); |
| 958 | cb(cur, "ffn_gate_b", il); |
| 959 | } |
| 960 | |
| 961 | if (gate_s) { |
| 962 | cur = ggml_mul(ctx0, cur, gate_s); |
| 963 | cb(cur, "ffn_gate_s", il); |
| 964 | } |
| 965 | |
| 966 | } else { |
| 967 | cur = tmp; |
| 968 | } |
| 969 | |
| 970 | switch (type_op) { |
| 971 | case LLM_FFN_SILU: |
nothing calls this directly
no test coverage detected