Helper function that tries to draft a token from only the static ngram cache:
| 64 | |
| 65 | // Helper function that tries to draft a token from only the static ngram cache: |
| 66 | static llama_token try_draft(common_ngram_cache & nc_static, const common_ngram ngram_static) { |
| 67 | common_ngram_cache::iterator part_static_it = nc_static.find(ngram_static); |
| 68 | if (part_static_it == nc_static.end()) { |
| 69 | return LLAMA_TOKEN_NULL; |
| 70 | } |
| 71 | const common_ngram_cache_part part_static = part_static_it->second; |
| 72 | |
| 73 | int max_count_static = 0; |
| 74 | int sum_count_static = 0; |
| 75 | llama_token max_token = LLAMA_TOKEN_NULL; |
| 76 | |
| 77 | for (std::pair<llama_token, int> token_count_static : part_static) { |
| 78 | const llama_token token = token_count_static.first; |
| 79 | const int32_t count_static = token_count_static.second; |
| 80 | |
| 81 | if (count_static > max_count_static) { |
| 82 | max_token = token; |
| 83 | max_count_static = count_static; |
| 84 | } |
| 85 | sum_count_static += count_static; |
| 86 | } |
| 87 | |
| 88 | if (sum_count_static < draft_min_sample_size_lax[LLAMA_NGRAM_STATIC-1]) { |
| 89 | return LLAMA_TOKEN_NULL; |
| 90 | } |
| 91 | if (100*max_count_static < draft_min_percent_lax[LLAMA_NGRAM_STATIC-1]*sum_count_static) { |
| 92 | return LLAMA_TOKEN_NULL; |
| 93 | } |
| 94 | return max_token; |
| 95 | } |
| 96 | |
| 97 | // Try to draft a token from primary cache (context/dynamic), validate with static cache: |
| 98 | static llama_token try_draft( |
no test coverage detected