| 256 | } |
| 257 | |
| 258 | void init_vision() { |
| 259 | GGML_ASSERT(ctx_v != nullptr); |
| 260 | image_preproc.reset(); |
| 261 | |
| 262 | projector_type proj = clip_get_projector_type(ctx_v); |
| 263 | |
| 264 | switch (proj) { |
| 265 | case PROJECTOR_TYPE_MLP: |
| 266 | case PROJECTOR_TYPE_MLP_NORM: |
| 267 | case PROJECTOR_TYPE_LDP: |
| 268 | case PROJECTOR_TYPE_LDPV2: |
| 269 | case PROJECTOR_TYPE_COGVLM: |
| 270 | case PROJECTOR_TYPE_JANUS_PRO: |
| 271 | case PROJECTOR_TYPE_GLM_EDGE: |
| 272 | { |
| 273 | bool has_pinpoints = !clip_get_hparams(ctx_v)->image_res_candidates.empty(); |
| 274 | if (has_pinpoints) { |
| 275 | image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v); |
| 276 | } else { |
| 277 | image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v); |
| 278 | } |
| 279 | } break; |
| 280 | case PROJECTOR_TYPE_MINICPMV: |
| 281 | { |
| 282 | int minicpmv_version = clip_is_minicpmv(ctx_v); |
| 283 | if (minicpmv_version == 2) { |
| 284 | // minicpmv 2.5 format: |
| 285 | // <image> (overview) </image><slice><image> (slice) </image><image> (slice) </image>\n ... </slice> |
| 286 | slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_5; |
| 287 | tok_ov_img_start = {lookup_token("<image>")}; |
| 288 | tok_ov_img_end = {lookup_token("</image>")}; |
| 289 | tok_slices_start = {lookup_token("<slice>")}; |
| 290 | tok_slices_end = {lookup_token("</slice>")}; |
| 291 | tok_sli_img_start = tok_ov_img_start; |
| 292 | tok_sli_img_end = tok_ov_img_end; |
| 293 | tok_row_end = {lookup_token("\n")}; |
| 294 | tok_row_end_trail = false; // no trailing end-of-row token |
| 295 | ov_img_first = true; |
| 296 | } else if (minicpmv_version == 3 || minicpmv_version == 4 || minicpmv_version == 5 || minicpmv_version == 6 || minicpmv_version == 100045) { |
| 297 | // minicpmv 2.6 format: |
| 298 | // <image> (overview) </image><slice> (slice) </slice><slice> (slice) </slice>\n ... |
| 299 | slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6; |
| 300 | tok_ov_img_start = {lookup_token("<image>")}; |
| 301 | tok_ov_img_end = {lookup_token("</image>")}; |
| 302 | tok_sli_img_start = {lookup_token("<slice>")}; |
| 303 | tok_sli_img_end = {lookup_token("</slice>")}; |
| 304 | tok_row_end = {lookup_token("\n")}; |
| 305 | tok_row_end_trail = false; // no trailing end-of-row token |
| 306 | ov_img_first = true; |
| 307 | |
| 308 | } else if (minicpmv_version != 0) { |
| 309 | throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version)); |
| 310 | } |
| 311 | image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v); |
| 312 | } break; |
| 313 | case PROJECTOR_TYPE_QWEN2VL: |
| 314 | case PROJECTOR_TYPE_QWEN25VL: |
| 315 | case PROJECTOR_TYPE_QWEN3VL: |
nothing calls this directly
no test coverage detected