Function encode_image_with_clip

examples/llava/llava.cpp:12–37

#include "base64.hpp"

static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {
    clip_image_f32 * img_res = make_clip_image_f32();
    if (!clip_image_preprocess(ctx_clip, img, img_res, /*pad2square =*/ true)) {
        fprintf(stderr, "%s: unable to preprocess image\n", __func__);
        clip_image_f32_free(img_res);
        return false;
    }

    *n_img_pos = clip_n_patches(ctx_clip);

    const int64_t t_img_enc_start_us = ggml_time_us();
    bool encoded = clip_image_encode(ctx_clip, n_threads, img_res, image_embd);
    clip_image_f32_free(img_res);
    if (!encoded) {
        fprintf(stderr, "Unable to encode image\n");

        return false;
    }

    const int64_t t_img_enc_end_us = ggml_time_us();
    float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;

    printf("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);

    return true;
}

bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip) {
        // make sure that the correct mmproj was used, i.e., compare apples to apples