Sample and get the output token.
| 220 | |
| 221 | // Sample and get the output token. |
| 222 | ErrNo sampleOutput(Graph &GraphRef, Context &CxtRef, |
| 223 | bool IsSingleTokenMode) noexcept { |
| 224 | // Use idx = -1 to sample the next token. |
| 225 | const llama_token Id = common_sampler_sample( |
| 226 | CxtRef.LlamaSampler, GraphRef.LlamaContext.get(), /* idx */ -1); |
| 227 | common_sampler_accept(CxtRef.LlamaSampler, Id, /* accept_grammar */ true); |
| 228 | |
| 229 | // Save the output token. |
| 230 | CxtRef.LlamaOutputTokens.emplace_back(Id); |
| 231 | std::string OutputString = |
| 232 | common_token_to_piece(GraphRef.LlamaContext.get(), Id); |
| 233 | CxtRef.LlamaOutputs.insert(CxtRef.LlamaOutputs.end(), OutputString.begin(), |
| 234 | OutputString.end()); |
| 235 | // In single token mode, we do not handle StreamStdout and ReversePrompt. |
| 236 | if (!IsSingleTokenMode) { |
| 237 | // When setting StreamStdout, we print the output to stdout. |
| 238 | if (CxtRef.Conf.StreamStdout) { |
| 239 | fmt::print("{}"sv, |
| 240 | common_token_to_piece(GraphRef.LlamaContext.get(), Id)); |
| 241 | std::fflush(stdout); |
| 242 | } |
| 243 | // Break if reverse prompt is found. |
| 244 | if (!CxtRef.Conf.ReversePrompt.empty() && |
| 245 | std::string(CxtRef.LlamaOutputs.begin(), CxtRef.LlamaOutputs.end()) |
| 246 | .find(CxtRef.Conf.ReversePrompt) != std::string::npos) { |
| 247 | LOG_INFO(GraphRef.EnableLog, "sampleOutput: reverse prompt found."sv) |
| 248 | return ErrNo::EndOfSequence; |
| 249 | } |
| 250 | } |
| 251 | // Deal with end of text token. |
| 252 | const llama_vocab *Vocab = llama_model_get_vocab(GraphRef.LlamaModel.get()); |
| 253 | // Only stop on EOS if GraphRef.Params.sampling.ignore_eos is false. |
| 254 | if (!GraphRef.Params.sampling.ignore_eos && |
| 255 | llama_vocab_is_eog(Vocab, common_sampler_last(CxtRef.LlamaSampler))) { |
| 256 | LOG_INFO(GraphRef.EnableLog, "sampleOutput: EOS token found."sv) |
| 257 | return ErrNo::EndOfSequence; |
| 258 | } |
| 259 | // Evaluate the output token. |
| 260 | return evaluateTokens(Span<const llama_token>(&Id, 1), GraphRef, |
| 261 | CxtRef.OutputBatch, CxtRef.NPos, true); |
| 262 | } |
| 263 | |
| 264 | #endif |
| 265 | } // namespace WasmEdge::Host::WASINN::GGML |