TODO: Merge into compute.
| 1784 | |
| 1785 | // TODO: Merge into compute. |
| 1786 | Expect<ErrNo> getEmbedding(Graph &GraphRef, Context &CxtRef) noexcept { |
| 1787 | LOG_DEBUG(GraphRef.EnableDebugLog, "getEmbedding"sv) |
| 1788 | |
| 1789 | const llama_token SepTokenId = llama_token_sep(GraphRef.LlamaModel.get()); |
| 1790 | if (SepTokenId > -1) { |
| 1791 | if (CxtRef.LlamaInputs.size() > 0 && |
| 1792 | CxtRef.LlamaInputs.back() != SepTokenId) { |
| 1793 | LOG_WARN( |
| 1794 | "getEmbedding: last token in the prompt is not SEP, "sv |
| 1795 | "'tokenizer.ggml.add_eos_token' should be set to 'true' in the GGUF "sv |
| 1796 | "header."sv) |
| 1797 | } |
| 1798 | } |
| 1799 | |
| 1800 | // Check if the input is too long. |
| 1801 | if (static_cast<int64_t>(CxtRef.LlamaInputs.size()) > |
| 1802 | GraphRef.Params.n_batch) { |
| 1803 | RET_ERROR( |
| 1804 | ErrNo::PromptTooLong, |
| 1805 | "getEmbedding: the prompt is too long. Your input has {} tokens exceeds batch "sv |
| 1806 | "size {}. Please reduce the input size or increase your batch-size."sv, |
| 1807 | CxtRef.LlamaInputs.size(), GraphRef.Params.n_batch) |
| 1808 | } |
| 1809 | |
| 1810 | // Evaluate the input tokens. |
| 1811 | auto ReturnCode = evaluateInput(GraphRef, CxtRef, "getEmbedding"sv); |
| 1812 | if (ReturnCode != ErrNo::Success) { |
| 1813 | return ReturnCode; |
| 1814 | } |
| 1815 | |
| 1816 | // Main prediction loop. |
| 1817 | const struct llama_model *LlamaModel = |
| 1818 | llama_get_model(GraphRef.LlamaContext.get()); |
| 1819 | const int32_t NEmbd = llama_n_embd(LlamaModel); |
| 1820 | std::vector<float> Embeddings(NEmbd); |
| 1821 | |
| 1822 | for (int I = 0; I < CxtRef.LlamaBatch.n_tokens; I++) { |
| 1823 | if (!CxtRef.LlamaBatch.logits[I]) { |
| 1824 | continue; |
| 1825 | } |
| 1826 | |
| 1827 | // Try to get sequence embeddings. |
| 1828 | auto *Embd = llama_get_embeddings_seq(GraphRef.LlamaContext.get(), |
| 1829 | CxtRef.LlamaBatch.seq_id[I][0]); |
| 1830 | if (Embd == nullptr) { |
| 1831 | Embd = llama_get_embeddings_ith(GraphRef.LlamaContext.get(), I); |
| 1832 | if (Embd == nullptr) { |
| 1833 | LOG_ERROR("getEmbedding: failed to get embeddings for token {}"sv, I); |
| 1834 | continue; |
| 1835 | } |
| 1836 | } |
| 1837 | |
| 1838 | // Normalize the embeddings. |
| 1839 | common_embd_normalize(Embd, Embeddings.data(), NEmbd, |
| 1840 | static_cast<int32_t>(CxtRef.Conf.EmbdNormalize)); |
| 1841 | } |
| 1842 | |
| 1843 | std::string EmbeddingString; |
no test coverage detected