| 3168 | } |
| 3169 | |
| 3170 | int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * img) { |
| 3171 | const auto & params = ctx->model.hparams; |
| 3172 | |
| 3173 | // for models with fixed size image, the input image is already pre-processed and resized to square |
| 3174 | int patch_size = params.patch_size; |
| 3175 | int n_patches = (img->nx / patch_size) * (img->ny / patch_size); |
| 3176 | |
| 3177 | projector_type proj = ctx->proj_type(); |
| 3178 | |
| 3179 | switch (proj) { |
| 3180 | case PROJECTOR_TYPE_MLP: |
| 3181 | case PROJECTOR_TYPE_MLP_NORM: |
| 3182 | case PROJECTOR_TYPE_JANUS_PRO: |
| 3183 | { |
| 3184 | // do nothing |
| 3185 | } break; |
| 3186 | case PROJECTOR_TYPE_LDP: |
| 3187 | case PROJECTOR_TYPE_LDPV2: |
| 3188 | case PROJECTOR_TYPE_GLM_EDGE: |
| 3189 | { |
| 3190 | n_patches /= 4; |
| 3191 | if (ctx->model.mm_boi) { |
| 3192 | n_patches += 2; // for BOI and EOI token embeddings |
| 3193 | } |
| 3194 | } break; |
| 3195 | case PROJECTOR_TYPE_MINICPMV: |
| 3196 | { |
| 3197 | // Use actual config value if available, otherwise fall back to hardcoded values |
| 3198 | if (params.minicpmv_query_num > 0) { |
| 3199 | n_patches = params.minicpmv_query_num; |
| 3200 | } else { |
| 3201 | // Fallback to hardcoded values for legacy models |
| 3202 | if (params.minicpmv_version == 2) { |
| 3203 | n_patches = 96; |
| 3204 | } else if (params.minicpmv_version == 3) { |
| 3205 | n_patches = 64; |
| 3206 | } else if (params.minicpmv_version == 4) { |
| 3207 | n_patches = 64; |
| 3208 | } else if (params.minicpmv_version == 5) { |
| 3209 | // MiniCPM-V 4.0 |
| 3210 | n_patches = 64; |
| 3211 | } else if (params.minicpmv_version == 6) { |
| 3212 | // MiniCPM-V 4.5 |
| 3213 | n_patches = 64; |
| 3214 | } else if (params.minicpmv_version == 100045) { |
| 3215 | // MiniCPM-o 4.5 |
| 3216 | n_patches = 64; |
| 3217 | } else { |
| 3218 | GGML_ABORT("Unknown minicpmv version"); |
| 3219 | } |
| 3220 | } |
| 3221 | } break; |
| 3222 | case PROJECTOR_TYPE_QWEN2VL: |
| 3223 | case PROJECTOR_TYPE_QWEN25VL: |
| 3224 | case PROJECTOR_TYPE_QWEN3VL: |
| 3225 | case PROJECTOR_TYPE_GLM4V: |
| 3226 | case PROJECTOR_TYPE_YOUTUVL: |
| 3227 | { |
no test coverage detected