| 494 | } |
| 495 | |
| 496 | ggml_tensor * clip_graph::build_ffn( |
| 497 | ggml_tensor * cur, |
| 498 | ggml_tensor * up, |
| 499 | ggml_tensor * up_b, |
| 500 | ggml_tensor * gate, |
| 501 | ggml_tensor * gate_b, |
| 502 | ggml_tensor * down, |
| 503 | ggml_tensor * down_b, |
| 504 | ffn_op_type type_op, |
| 505 | int il) const { |
| 506 | |
| 507 | ggml_tensor * tmp = up ? ggml_mul_mat(ctx0, up, cur) : cur; |
| 508 | cb(tmp, "ffn_up", il); |
| 509 | |
| 510 | if (up_b) { |
| 511 | tmp = ggml_add(ctx0, tmp, up_b); |
| 512 | cb(tmp, "ffn_up_b", il); |
| 513 | } |
| 514 | |
| 515 | if (gate) { |
| 516 | cur = ggml_mul_mat(ctx0, gate, cur); |
| 517 | cb(cur, "ffn_gate", il); |
| 518 | |
| 519 | if (gate_b) { |
| 520 | cur = ggml_add(ctx0, cur, gate_b); |
| 521 | cb(cur, "ffn_gate_b", il); |
| 522 | } |
| 523 | } else { |
| 524 | cur = tmp; |
| 525 | } |
| 526 | |
| 527 | // we only support parallel ffn for now |
| 528 | switch (type_op) { |
| 529 | case FFN_SILU: |
| 530 | if (gate) { |
| 531 | cur = ggml_swiglu_split(ctx0, cur, tmp); |
| 532 | cb(cur, "ffn_swiglu", il); |
| 533 | } else { |
| 534 | cur = ggml_silu(ctx0, cur); |
| 535 | cb(cur, "ffn_silu", il); |
| 536 | } break; |
| 537 | case FFN_GELU: |
| 538 | if (gate) { |
| 539 | cur = ggml_geglu_split(ctx0, cur, tmp); |
| 540 | cb(cur, "ffn_geglu", il); |
| 541 | } else { |
| 542 | cur = ggml_gelu(ctx0, cur); |
| 543 | cb(cur, "ffn_gelu", il); |
| 544 | } break; |
| 545 | case FFN_GELU_ERF: |
| 546 | if (gate) { |
| 547 | cur = ggml_geglu_erf_split(ctx0, cur, tmp); |
| 548 | cb(cur, "ffn_geglu_erf", il); |
| 549 | } else { |
| 550 | cur = ggml_gelu_erf(ctx0, cur); |
| 551 | cb(cur, "ffn_gelu_erf", il); |
| 552 | } break; |
| 553 | case FFN_GELU_QUICK: |
nothing calls this directly
no test coverage detected