| 1 | #include "models.h" |
| 2 | |
| 3 | ggml_cgraph * clip_graph_qwen2vl::build() { |
| 4 | GGML_ASSERT(model.patch_bias == nullptr); |
| 5 | GGML_ASSERT(model.class_embedding == nullptr); |
| 6 | |
| 7 | const int batch_size = 1; |
| 8 | const bool use_window_attn = hparams.n_wa_pattern > 0; |
| 9 | const int n_wa_pattern = hparams.n_wa_pattern; |
| 10 | const int n_pos = n_patches; |
| 11 | const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position |
| 12 | |
| 13 | norm_type norm_t = proj_type == PROJECTOR_TYPE_QWEN25VL |
| 14 | ? NORM_TYPE_RMS // qwen 2.5 vl |
| 15 | : NORM_TYPE_NORMAL; // qwen 2 vl |
| 16 | |
| 17 | int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; |
| 18 | |
| 19 | ggml_tensor * inp_raw = build_inp_raw(); |
| 20 | ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); |
| 21 | |
| 22 | GGML_ASSERT(img.nx % (patch_size * 2) == 0); |
| 23 | GGML_ASSERT(img.ny % (patch_size * 2) == 0); |
| 24 | |
| 25 | // second conv dimension |
| 26 | { |
| 27 | auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1); |
| 28 | inp = ggml_add(ctx0, inp, inp_1); |
| 29 | |
| 30 | inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] |
| 31 | inp = ggml_cont_4d( |
| 32 | ctx0, inp, |
| 33 | n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); |
| 34 | inp = ggml_reshape_4d( |
| 35 | ctx0, inp, |
| 36 | n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); |
| 37 | inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); |
| 38 | inp = ggml_cont_3d( |
| 39 | ctx0, inp, |
| 40 | n_embd, n_patches_x * n_patches_y, batch_size); |
| 41 | } |
| 42 | |
| 43 | ggml_tensor * inpL = inp; |
| 44 | ggml_tensor * window_mask = nullptr; |
| 45 | ggml_tensor * window_idx = nullptr; |
| 46 | ggml_tensor * inv_window_idx = nullptr; |
| 47 | |
| 48 | ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids); |
| 49 | ggml_set_name(positions, "positions"); |
| 50 | ggml_set_input(positions); |
| 51 | |
| 52 | // pre-layernorm |
| 53 | if (model.pre_ln_w) { |
| 54 | inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1); |
| 55 | } |
| 56 | |
| 57 | if (use_window_attn) { |
| 58 | // handle window attention inputs |
| 59 | inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4); |
| 60 | ggml_set_name(inv_window_idx, "inv_window_idx"); |
nothing calls this directly
no test coverage detected