| 133 | // TODO @ngxson : add timings |
| 134 | |
| 135 | mtmd_context(const char * mmproj_fname, |
| 136 | const llama_model * text_model, |
| 137 | const mtmd_context_params & ctx_params) : |
| 138 | text_model (text_model), |
| 139 | print_timings(ctx_params.print_timings), |
| 140 | n_threads (ctx_params.n_threads), |
| 141 | media_marker (ctx_params.media_marker), |
| 142 | n_embd_text (llama_model_n_embd(text_model)) |
| 143 | { |
| 144 | if (std::string(ctx_params.image_marker) != MTMD_DEFAULT_IMAGE_MARKER) { |
| 145 | throw std::runtime_error("custom image_marker is not supported anymore, use media_marker instead"); |
| 146 | } |
| 147 | |
| 148 | if (media_marker.empty()) { |
| 149 | throw std::runtime_error("media_marker must not be empty"); |
| 150 | } |
| 151 | |
| 152 | clip_context_params ctx_clip_params; |
| 153 | ctx_clip_params.use_gpu = ctx_params.use_gpu; |
| 154 | ctx_clip_params.verbosity = ctx_params.verbosity; |
| 155 | auto res = clip_init(mmproj_fname, ctx_clip_params); |
| 156 | ctx_v = res.ctx_v; |
| 157 | ctx_a = res.ctx_a; |
| 158 | if (!ctx_v && !ctx_a) { |
| 159 | throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname)); |
| 160 | } |
| 161 | |
| 162 | // if both vision and audio mmproj are present, we need to validate their n_embd |
| 163 | if (ctx_v && ctx_a) { |
| 164 | int n_embd_v = clip_n_mmproj_embd(ctx_v); |
| 165 | int n_embd_a = clip_n_mmproj_embd(ctx_a); |
| 166 | if (n_embd_v != n_embd_a) { |
| 167 | throw std::runtime_error(string_format( |
| 168 | "mismatch between vision and audio mmproj (n_embd_v = %d, n_embd_a = %d)\n", |
| 169 | n_embd_v, n_embd_a)); |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | // since we already validate n_embd of vision and audio mmproj, |
| 174 | // we can safely assume that they are the same |
| 175 | int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a); |
| 176 | if (n_embd_text != n_embd_clip) { |
| 177 | throw std::runtime_error(string_format( |
| 178 | "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n" |
| 179 | "hint: you may be using wrong mmproj\n", |
| 180 | n_embd_text, n_embd_clip)); |
| 181 | } |
| 182 | if (ctx_v) { |
| 183 | init_vision(); |
| 184 | } |
| 185 | if (ctx_a) { |
| 186 | init_audio(); |
| 187 | } |
| 188 | } |
| 189 | |
| 190 | void init_vision() { |
| 191 | GGML_ASSERT(ctx_v != nullptr); |
nothing calls this directly
no test coverage detected