| 1005 | } |
| 1006 | |
| 1007 | int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk) { |
| 1008 | if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) { |
| 1009 | LOG_WRN("mtmd_encode_chunk has no effect for text chunks\n"); |
| 1010 | return 0; |
| 1011 | } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) { |
| 1012 | if (!ctx->ctx_v) { |
| 1013 | LOG_ERR("%s: model does not support vision input\n", __func__); |
| 1014 | return 1; |
| 1015 | } |
| 1016 | return mtmd_encode(ctx, chunk->tokens_image.get()); |
| 1017 | } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { |
| 1018 | if (!ctx->ctx_a) { |
| 1019 | LOG_ERR("%s: model does not support audio input\n", __func__); |
| 1020 | return 1; |
| 1021 | } |
| 1022 | int n_mmproj_embd = ctx->n_embd_text; |
| 1023 | ctx->image_embd_v.resize(chunk->tokens_audio->n_tokens * n_mmproj_embd); |
| 1024 | bool ok = clip_image_batch_encode( |
| 1025 | ctx->ctx_a, |
| 1026 | ctx->n_threads, |
| 1027 | &chunk->tokens_audio->batch_f32, |
| 1028 | ctx->image_embd_v.data()); |
| 1029 | return ok ? 0 : 1; |
| 1030 | } |
| 1031 | |
| 1032 | LOG_ERR("%s: unknown chunk type %d\n", __func__, (int)chunk->type); |
| 1033 | return 1; |
| 1034 | } |
| 1035 | |
| 1036 | int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) { |
| 1037 | clip_ctx * ctx_clip = ctx->ctx_v; |
no test coverage detected