| 1 | #include "models.h" |
| 2 | |
| 3 | ggml_cgraph * clip_graph_glm4v::build() { |
| 4 | GGML_ASSERT(model.patch_bias != nullptr); |
| 5 | GGML_ASSERT(model.position_embeddings != nullptr); |
| 6 | GGML_ASSERT(model.class_embedding == nullptr); |
| 7 | |
| 8 | const int batch_size = 1; |
| 9 | |
| 10 | norm_type norm_t = NORM_TYPE_RMS; |
| 11 | |
| 12 | ggml_tensor * inp_raw = build_inp_raw(); |
| 13 | ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1); |
| 14 | |
| 15 | int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; |
| 16 | ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches * 4); |
| 17 | ggml_set_name(positions, "positions"); |
| 18 | ggml_set_input(positions); |
| 19 | |
| 20 | GGML_ASSERT(img.nx % (patch_size * 2) == 0); |
| 21 | GGML_ASSERT(img.ny % (patch_size * 2) == 0); |
| 22 | |
| 23 | // second conv dimension |
| 24 | { |
| 25 | auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1); |
| 26 | inp = ggml_add(ctx0, inp, inp_1); |
| 27 | |
| 28 | inp = ggml_permute(ctx0, inp, 1, 2, 0, 3); // [w, h, c, b] -> [c, w, h, b] |
| 29 | inp = ggml_cont_4d( |
| 30 | ctx0, inp, |
| 31 | n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); |
| 32 | inp = ggml_reshape_4d( |
| 33 | ctx0, inp, |
| 34 | n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); |
| 35 | inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); |
| 36 | inp = ggml_cont_3d( |
| 37 | ctx0, inp, |
| 38 | n_embd, n_patches_x * n_patches_y, batch_size); |
| 39 | } |
| 40 | |
| 41 | // add patch bias |
| 42 | inp = ggml_add(ctx0, inp, model.patch_bias); |
| 43 | cb(inp, "patch_bias", -1); |
| 44 | |
| 45 | // pos-conv norm |
| 46 | inp = build_norm(inp, model.norm_embd_w, model.norm_embd_b, norm_t, eps, -1); |
| 47 | |
| 48 | // calculate absolute position embedding and apply |
| 49 | ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BICUBIC); |
| 50 | learned_pos_embd = ggml_cont_4d( |
| 51 | ctx0, learned_pos_embd, |
| 52 | n_embd * 2, n_patches_x / 2, n_patches_y, batch_size); |
| 53 | learned_pos_embd = ggml_reshape_4d( |
| 54 | ctx0, learned_pos_embd, |
| 55 | n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2)); |
| 56 | learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3); |
| 57 | learned_pos_embd = ggml_cont_3d( |
| 58 | ctx0, learned_pos_embd, |
| 59 | n_embd, n_patches_x * n_patches_y, batch_size); |
| 60 | cb(learned_pos_embd, "learned_pos_embd", -1); |
nothing calls this directly
no test coverage detected