| 1450 | } |
| 1451 | |
| 1452 | static bool needs_raw_logits(const llama_ubatch & ubatch, const std::map<llama_seq_id, llama_sampler *> & samplers) { |
| 1453 | for (uint32_t i = 0; i < ubatch.n_tokens; i++) { |
| 1454 | if (!ubatch.output[i]) { |
| 1455 | continue; |
| 1456 | } |
| 1457 | |
| 1458 | // Check if the output token has at least one sequence without a backend sampler. |
| 1459 | for (int32_t j = 0; j < ubatch.n_seq_id[i]; ++j) { |
| 1460 | llama_seq_id seq_id = ubatch.seq_id[i][j]; |
| 1461 | if (samplers.find(seq_id) == samplers.end()) { |
| 1462 | return true; |
| 1463 | } |
| 1464 | } |
| 1465 | } |
| 1466 | return false; // all sequences use backend sampling |
| 1467 | } |
| 1468 | |
| 1469 | int llama_context::decode(const llama_batch & batch_inp) { |
| 1470 | GGML_ASSERT((!batch_inp.token && batch_inp.embd) || (batch_inp.token && !batch_inp.embd)); // NOLINT |