| 82 | } |
| 83 | |
| 84 | static void batch_process(llama_context * ctx, llama_batch & batch, float * output, int n_seq, int n_embd) { |
| 85 | // clear previous kv_cache values (irrelevant for embeddings) |
| 86 | llama_kv_self_clear(ctx); |
| 87 | |
| 88 | // run model |
| 89 | LOG_INF("%s: n_tokens = %d, n_seq = %d\n", __func__, batch.n_tokens, n_seq); |
| 90 | if (llama_decode(ctx, batch) < 0) { |
| 91 | LOG_ERR("%s : failed to process\n", __func__); |
| 92 | } |
| 93 | |
| 94 | for (int i = 0; i < batch.n_tokens; i++) { |
| 95 | if (!batch.logits[i]) { |
| 96 | continue; |
| 97 | } |
| 98 | |
| 99 | // try to get sequence embeddings - supported only when pooling_type is not NONE |
| 100 | const float * embd = llama_get_embeddings_seq(ctx, batch.seq_id[i][0]); |
| 101 | if (embd == NULL) { |
| 102 | embd = llama_get_embeddings_ith(ctx, i); |
| 103 | if (embd == NULL) { |
| 104 | LOG_ERR("%s: failed to get embeddings for token %d\n", __func__, i); |
| 105 | continue; |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | float * out = output + batch.seq_id[i][0] * n_embd; |
| 110 | common_embd_normalize(embd, out, n_embd, 2); |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | int main(int argc, char ** argv) { |
| 115 | common_params params; |
no test coverage detected