| 1 | #include "models.h" |
| 2 | |
| 3 | ggml_cgraph * clip_graph_qwen3vl::build() { |
| 4 | GGML_ASSERT(model.patch_bias != nullptr); |
| 5 | GGML_ASSERT(model.position_embeddings != nullptr); |
| 6 | GGML_ASSERT(model.class_embedding == nullptr); |
| 7 | |
| 8 | const int batch_size = 1; |
| 9 | const int n_pos = n_patches; |
| 10 | const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position |
| 11 | |
| 12 | norm_type norm_t = NORM_TYPE_NORMAL; |
| 13 | |
| 14 | int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; |
| 15 | |
| 16 | ggml_tensor * inp_raw = build_inp_raw(); |
| 17 | ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); |
| 18 | |
| 19 | GGML_ASSERT(img.nx % (patch_size * 2) == 0); |
| 20 | GGML_ASSERT(img.ny % (patch_size * 2) == 0); |
| 21 | |
| 22 | // second conv dimension |
| 23 | { |
| 24 | auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1); |
| 25 | inp = ggml_add(ctx0, inp, inp_1); |
| 26 | |
| 27 | inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] |
| 28 | inp = ggml_cont_4d( |
| 29 | ctx0, inp, |
| 30 | n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); |
| 31 | inp = ggml_reshape_4d( |
| 32 | ctx0, inp, |
| 33 | n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); |
| 34 | inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); |
| 35 | inp = ggml_cont_3d( |
| 36 | ctx0, inp, |
| 37 | n_embd, n_patches_x * n_patches_y, batch_size); |
| 38 | } |
| 39 | |
| 40 | // add patch bias |
| 41 | if (model.patch_bias != nullptr) { |
| 42 | inp = ggml_add(ctx0, inp, model.patch_bias); |
| 43 | cb(inp, "patch_bias", -1); |
| 44 | } |
| 45 | |
| 46 | // calculate absolute position embedding and apply |
| 47 | ggml_tensor * learned_pos_embd = resize_position_embeddings(); |
| 48 | learned_pos_embd = ggml_cont_4d( |
| 49 | ctx0, learned_pos_embd, |
| 50 | n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); |
| 51 | learned_pos_embd = ggml_reshape_4d( |
| 52 | ctx0, learned_pos_embd, |
| 53 | n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); |
| 54 | learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3); |
| 55 | learned_pos_embd = ggml_cont_3d( |
| 56 | ctx0, learned_pos_embd, |
| 57 | n_embd, n_patches_x * n_patches_y, batch_size); |
| 58 | inp = ggml_add(ctx0, inp, learned_pos_embd); |
| 59 | cb(inp, "inp_pos_emb", -1); |
| 60 |
nothing calls this directly
no test coverage detected