| 430 | } |
| 431 | |
| 432 | bool server_tokens::validate(const struct llama_context * ctx) const { |
| 433 | const llama_model * model = llama_get_model(ctx); |
| 434 | const llama_vocab * vocab = llama_model_get_vocab(model); |
| 435 | const int32_t n_vocab = llama_vocab_n_tokens(vocab); |
| 436 | |
| 437 | for (size_t i = 0; i < tokens.size(); ++i) { |
| 438 | const auto & t = tokens[i]; |
| 439 | if (t == LLAMA_TOKEN_NULL) { |
| 440 | try { |
| 441 | const auto & chunk = find_chunk(i); |
| 442 | size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk.get()); |
| 443 | i += n_tokens - 1; // will be +1 by the for loop |
| 444 | } catch (const std::exception & e) { |
| 445 | return false; |
| 446 | } |
| 447 | } else if (t < 0 || t >= n_vocab) { |
| 448 | return false; |
| 449 | } |
| 450 | } |
| 451 | return true; |
| 452 | } |
| 453 | |
| 454 | int32_t server_tokens::process_chunk( |
| 455 | llama_context * ctx, |
nothing calls this directly
no test coverage detected