| 2062 | } |
| 2063 | |
| 2064 | void load_hparams(clip_model & model, clip_modality modality) { |
| 2065 | auto & hparams = model.hparams; |
| 2066 | std::string log_ffn_op; // for logging |
| 2067 | |
| 2068 | // sanity check |
| 2069 | if (modality == CLIP_MODALITY_VISION) { |
| 2070 | GGML_ASSERT(has_vision); |
| 2071 | } else if (modality == CLIP_MODALITY_AUDIO) { |
| 2072 | GGML_ASSERT(has_audio); |
| 2073 | } |
| 2074 | model.modality = modality; |
| 2075 | |
| 2076 | |
| 2077 | // projector type |
| 2078 | std::string proj_type; |
| 2079 | { |
| 2080 | get_string(KEY_PROJ_TYPE, proj_type, false); |
| 2081 | if (!proj_type.empty()) { |
| 2082 | model.proj_type = clip_projector_type_from_string(proj_type); |
| 2083 | } |
| 2084 | if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) { |
| 2085 | throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str())); |
| 2086 | } |
| 2087 | |
| 2088 | // correct arch for multimodal models |
| 2089 | if (model.proj_type == PROJECTOR_TYPE_QWEN25O) { |
| 2090 | model.proj_type = modality == CLIP_MODALITY_VISION |
| 2091 | ? PROJECTOR_TYPE_QWEN25VL |
| 2092 | : PROJECTOR_TYPE_QWEN2A; |
| 2093 | } |
| 2094 | } |
| 2095 | |
| 2096 | const bool is_vision = model.modality == CLIP_MODALITY_VISION; |
| 2097 | const bool is_audio = model.modality == CLIP_MODALITY_AUDIO; |
| 2098 | |
| 2099 | // other hparams |
| 2100 | { |
| 2101 | const char * prefix = is_vision ? "vision" : "audio"; |
| 2102 | get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd); |
| 2103 | get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head); |
| 2104 | get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff); |
| 2105 | get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer); |
| 2106 | get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim); |
| 2107 | get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps); |
| 2108 | |
| 2109 | if (is_vision) { |
| 2110 | get_u32(KEY_IMAGE_SIZE, hparams.image_size); |
| 2111 | get_u32(KEY_PATCH_SIZE, hparams.patch_size); |
| 2112 | get_u32(KEY_IMAGE_CROP_RESOLUTION, hparams.image_crop_resolution, false); |
| 2113 | get_arr_int(KEY_IMAGE_GRID_PINPOINTS, hparams.image_grid_pinpoints, false); |
| 2114 | get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy |
| 2115 | |
| 2116 | } else if (is_audio) { |
| 2117 | get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins); |
| 2118 | |
| 2119 | } else { |
| 2120 | GGML_ASSERT(false && "unknown modality"); |
| 2121 | } |
no test coverage detected