Function clip_image_preprocess

subprojects/llama.cpp/tools/mtmd/clip.cpp:2812–3094

returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector res_imgs memory is being allocated here, previous allocations will be freed if found

// returns the normalized float tensor for llava-1.5, for spatial_unpad with anyres processing for llava-1.6 it returns the normalized image patch tensors as a vector
// res_imgs memory is being allocated here, previous allocations will be freed if found
bool clip_image_preprocess(struct clip_ctx * ctx, const clip_image_u8 * img, struct clip_image_f32_batch * res_imgs) {
    clip_image_size original_size{img->nx, img->ny};
    auto & params = ctx->model.hparams;

    switch (ctx->proj_type()) {
        case PROJECTOR_TYPE_MINICPMV:
            {
                auto const inst = llava_uhd::get_slice_instructions(ctx, original_size);
                std::vector<clip_image_u8_ptr> imgs = llava_uhd::slice_image(img, inst);

                for (size_t i = 0; i < imgs.size(); ++i) {
                    // clip_image_save_to_bmp(*imgs[i], "slice_" + std::to_string(i) + ".bmp");
                    clip_image_f32_ptr res(clip_image_f32_init());
                    normalize_image_u8_to_f32(*imgs[i], *res, params.image_mean, params.image_std);
                    res_imgs->entries.push_back(std::move(res));
                }

                res_imgs->grid_x = inst.grid_size.width;
                res_imgs->grid_y = inst.grid_size.height;
            } break;

        case PROJECTOR_TYPE_QWEN2VL:
        case PROJECTOR_TYPE_QWEN25VL:
        case PROJECTOR_TYPE_QWEN3VL:
        case PROJECTOR_TYPE_GLM4V:
            {
                GGML_ASSERT(params.image_min_pixels > 0 && params.image_max_pixels > 0);
                clip_image_u8 resized;
                const clip_image_size new_size = img_tool::calc_size_preserved_ratio(
                    original_size,
                    params.patch_size * 2,
                    params.image_min_pixels,
                    params.image_max_pixels);
                img_tool::resize(*img, resized, new_size, img_tool::RESIZE_ALGO_BILINEAR, false);
                // clip_image_save_to_bmp(resized, "preproc.bmp");
                clip_image_f32_ptr img_f32(clip_image_f32_init());
                // clip_image_f32_ptr res(clip_image_f32_init());
                normalize_image_u8_to_f32(resized, *img_f32, params.image_mean, params.image_std);
                // res_imgs->data[0] = *res;
                res_imgs->entries.push_back(std::move(img_f32));
            } break;
        case PROJECTOR_TYPE_YOUTUVL:
            {
                const int patch_size = params.patch_size;  // typically 16
                const int merge_size = params.n_merge;      // typically 2
                const int align_size = patch_size * merge_size;  // 32

                const int max_num_patches = params.image_max_pixels > 0 ?
                    params.image_max_pixels / (patch_size * patch_size) : 256;

                // Linear search for optimal scale to fit within max_num_patches
                float scale = 1.0f;
                int target_height = original_size.height;
                int target_width = original_size.width;

                auto get_scaled_image_size = [align_size](float scale, int size) -> int {
                    float scaled_size = size * scale;
                    // Round up to nearest multiple of align_size