| 35 | } |
| 36 | |
| 37 | static void batch_decode(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd_out, int embd_norm) { |
| 38 | const enum llama_pooling_type pooling_type = llama_pooling_type(ctx); |
| 39 | |
| 40 | // clear previous kv_cache values (irrelevant for embeddings) |
| 41 | llama_memory_clear(llama_get_memory(ctx), true); |
| 42 | |
| 43 | // run model |
| 44 | LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); |
| 45 | if (llama_decode(ctx, batch) < 0) { |
| 46 | LOG_ERR("%s : failed to process\n", __func__); |
| 47 | } |
| 48 | |
| 49 | for (int i = 0; i < batch.n_tokens; i++) { |
| 50 | if (!batch.logits[i]) { |
| 51 | continue; |
| 52 | } |
| 53 | |
| 54 | const float * embd = nullptr; |
| 55 | int embd_pos = 0; |
| 56 | |
| 57 | if (pooling_type == LLAMA_POOLING_TYPE_NONE) { |
| 58 | // try to get token embeddings |
| 59 | embd = llama_get_embeddings_ith(ctx, i); |
| 60 | embd_pos = i; |
| 61 | GGML_ASSERT(embd != NULL && "failed to get token embeddings"); |
| 62 | } else { |
| 63 | // try to get sequence embeddings - supported only when pooling_type is not NONE |
| 64 | embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); |
| 65 | embd_pos = batch.seq_id[i][0]; |
| 66 | GGML_ASSERT(embd != NULL && "failed to get sequence embeddings"); |
| 67 | } |
| 68 | |
| 69 | float * out = output + embd_pos * n_embd_out; |
| 70 | common_embd_normalize(embd, out, n_embd_out, embd_norm); |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | // plain, pipe-friendly output: one embedding per line |
| 75 | static void print_raw_embeddings(const float * emb, |
no test coverage detected