| 1 | #include "models.h" |
| 2 | |
| 3 | ggml_cgraph * clip_graph_conformer::build() { |
| 4 | const int n_frames = img.nx; |
| 5 | const int n_pos = n_frames / 2; |
| 6 | const int n_pos_embd = (((((n_frames + 1) / 2) + 1) / 2 + 1) / 2) * 2 - 1; |
| 7 | GGML_ASSERT(model.position_embeddings->ne[1] >= n_pos); |
| 8 | |
| 9 | ggml_tensor * pos_emb = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 512, n_pos_embd); |
| 10 | ggml_set_name(pos_emb, "pos_emb"); |
| 11 | ggml_set_input(pos_emb); |
| 12 | ggml_build_forward_expand(gf, pos_emb); |
| 13 | |
| 14 | ggml_tensor * inp = build_inp_raw(1); |
| 15 | |
| 16 | auto * cur = ggml_cont(ctx0, ggml_transpose(ctx0, inp)); |
| 17 | |
| 18 | // pre encode, conv subsampling |
| 19 | { |
| 20 | // layer.0 - conv2d |
| 21 | cur = ggml_conv_2d(ctx0, model.pre_encode_conv_X_w[0], cur, 2, 2, 1, 1, 1, 1); |
| 22 | cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[0]); |
| 23 | cb(cur, "conformer.pre_encode.conv.{}", 0); |
| 24 | |
| 25 | // layer.1 - relu |
| 26 | cur = ggml_relu_inplace(ctx0, cur); |
| 27 | |
| 28 | // layer.2 conv2d dw |
| 29 | cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[2], cur, 2, 2, 1, 1, 1, 1); |
| 30 | cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[2]); |
| 31 | cb(cur, "conformer.pre_encode.conv.{}", 2); |
| 32 | |
| 33 | // layer.3 conv2d |
| 34 | cur = ggml_conv_2d_direct(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1); |
| 35 | cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]); |
| 36 | cb(cur, "conformer.pre_encode.conv.{}", 3); |
| 37 | |
| 38 | // layer.4 - relu |
| 39 | cur = ggml_relu_inplace(ctx0, cur); |
| 40 | |
| 41 | // layer.5 conv2d dw |
| 42 | cur = ggml_conv_2d_dw_direct(ctx0, model.pre_encode_conv_X_w[5], cur, 2, 2, 1, 1, 1, 1); |
| 43 | cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[5]); |
| 44 | cb(cur, "conformer.pre_encode.conv.{}", 5); |
| 45 | |
| 46 | // layer.6 conv2d |
| 47 | cur = ggml_conv_2d_direct(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1); |
| 48 | cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]); |
| 49 | cb(cur, "conformer.pre_encode.conv.{}", 6); |
| 50 | |
| 51 | // layer.7 - relu |
| 52 | cur = ggml_relu_inplace(ctx0, cur); |
| 53 | |
| 54 | // flatten channel and frequency axis |
| 55 | cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 0, 2, 1, 3)); |
| 56 | cur = ggml_reshape_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2]); |
| 57 | |
| 58 | // calculate out |
| 59 | cur = ggml_mul_mat(ctx0, model.pre_encode_out_w, cur); |
| 60 | cur = ggml_add(ctx0, cur, model.pre_encode_out_b); |
nothing calls this directly
no test coverage detected