| 1 | #include "models.h" |
| 2 | |
| 3 | ggml_cgraph * clip_graph_youtuvl::build() { |
| 4 | GGML_ASSERT(model.class_embedding == nullptr); |
| 5 | const int batch_size = 1; |
| 6 | const bool use_window_attn = !hparams.wa_layer_indexes.empty(); |
| 7 | const int n_pos = n_patches; |
| 8 | const int num_position_ids = n_pos * 4; |
| 9 | const int m = 2; |
| 10 | const int Wp = n_patches_x; |
| 11 | const int Hp = n_patches_y; |
| 12 | const int Hm = Hp / m; |
| 13 | const int Wm = Wp / m; |
| 14 | norm_type norm_t = NORM_TYPE_NORMAL; |
| 15 | |
| 16 | int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4}; |
| 17 | |
| 18 | ggml_tensor * inp = build_inp_raw(); |
| 19 | |
| 20 | // change conv3d to linear |
| 21 | // reshape and permute to get patches, permute from (patch_size, m, Wm, patch_size, m, Hm, C) to (C, patch_size, patch_size, m, m, Wm, Hm) |
| 22 | { |
| 23 | inp = ggml_reshape_4d( |
| 24 | ctx0, inp, |
| 25 | Wm * m * patch_size, m * patch_size, Hm, 3); |
| 26 | inp = ggml_permute(ctx0, inp, 1, 2, 3, 0); |
| 27 | inp = ggml_cont_4d( |
| 28 | ctx0, inp, |
| 29 | m * patch_size * 3, Wm, m * patch_size, Hm); |
| 30 | |
| 31 | inp = ggml_permute(ctx0, inp, 0, 2, 1, 3); |
| 32 | inp = ggml_cont_4d( |
| 33 | ctx0, inp, |
| 34 | m * patch_size * 3, patch_size, m, Hm * Wm); |
| 35 | |
| 36 | inp = ggml_permute(ctx0, inp, 1, 0, 2, 3); |
| 37 | inp = ggml_cont_4d( |
| 38 | ctx0, inp, |
| 39 | patch_size, 3, patch_size, Hm * Wm * m * m); |
| 40 | |
| 41 | inp = ggml_permute(ctx0, inp, 2, 0, 1, 3); |
| 42 | inp = ggml_cont_3d( |
| 43 | ctx0, inp, |
| 44 | 3*patch_size* patch_size, Hm * Wm * m * m, 1); |
| 45 | } |
| 46 | inp = ggml_mul_mat(ctx0, model.patch_embeddings_0, inp); |
| 47 | |
| 48 | if (model.patch_bias) { |
| 49 | inp = ggml_add(ctx0, inp, model.patch_bias); |
| 50 | } |
| 51 | |
| 52 | inp = ggml_reshape_2d(ctx0, inp, n_embd, n_patches); |
| 53 | |
| 54 | ggml_tensor * inpL = inp; |
| 55 | ggml_tensor * window_mask = nullptr; |
| 56 | ggml_tensor * window_idx = nullptr; |
| 57 | ggml_tensor * inv_window_idx = nullptr; |
| 58 | |
| 59 | ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids); |
| 60 | ggml_set_name(positions, "positions"); |
nothing calls this directly
no test coverage detected