| 10 | #include <algorithm> |
| 11 | |
| 12 | void common_ngram_cache_update(common_ngram_cache & ngram_cache, int ngram_min, int ngram_max, |
| 13 | std::vector<llama_token> & inp, int nnew, bool print_progress) { |
| 14 | const int64_t t_start_ms = ggml_time_ms(); |
| 15 | const int64_t inp_size = inp.size(); |
| 16 | |
| 17 | const int64_t n_todo = inp_size * (ngram_max - ngram_min + 1); |
| 18 | int64_t n_done = 0; |
| 19 | |
| 20 | for (int64_t ngram_size = ngram_min; ngram_size <= ngram_max; ++ngram_size) { |
| 21 | const int64_t i_start = std::max(inp_size - nnew, ngram_size); |
| 22 | for (int64_t i = i_start; i < inp_size; ++i) { |
| 23 | const int64_t ngram_start = i - ngram_size; |
| 24 | common_ngram ngram(&inp[ngram_start], ngram_size); |
| 25 | const llama_token token = inp[i]; |
| 26 | |
| 27 | common_ngram_cache::iterator part_it = ngram_cache.find(ngram); |
| 28 | if (part_it == ngram_cache.end()) { |
| 29 | common_ngram_cache_part part; |
| 30 | part.emplace(token, 1); |
| 31 | ngram_cache.emplace(ngram, part); |
| 32 | } else { |
| 33 | common_ngram_cache_part::iterator token_count_it = part_it->second.find(token); |
| 34 | if (token_count_it == part_it->second.end()) { |
| 35 | part_it->second.emplace(token, 1); |
| 36 | } else { |
| 37 | token_count_it->second++; |
| 38 | } |
| 39 | } |
| 40 | ++n_done; |
| 41 | |
| 42 | if (print_progress && n_done % 10000000 == 0) { |
| 43 | const int64_t t_now_ms = ggml_time_ms(); |
| 44 | const int64_t eta_ms = (inp_size*(ngram_max-ngram_min+1) - n_done) * (t_now_ms - t_start_ms) / n_done; |
| 45 | const int64_t eta_min = eta_ms / (60*1000); |
| 46 | const int64_t eta_s = (eta_ms - 60*1000*eta_min) / 1000; |
| 47 | |
| 48 | fprintf(stderr, "%s: %" PRId64 "/%" PRId64 " done, ETA: %02" PRId64 ":%02" PRId64 "\n", __func__, n_done, n_todo, eta_min, eta_s); |
| 49 | } |
| 50 | } |
| 51 | } |
| 52 | } |
| 53 | |
| 54 | // Helper function to get a token from the combined, speculative sequence of inp and draft. |
| 55 | static llama_token get_token(const std::vector<llama_token> & inp, const std::vector<llama_token> & draft, const size_t i) { |