| 144 | } |
| 145 | |
| 146 | void common_ngram_cache_draft( |
| 147 | std::vector<llama_token> & inp, std::vector<llama_token> & draft, int n_draft, int ngram_min, int ngram_max, |
| 148 | common_ngram_cache & nc_context, common_ngram_cache & nc_dynamic, common_ngram_cache & nc_static |
| 149 | ) { |
| 150 | GGML_ASSERT(draft.size() == 1); |
| 151 | const int inp_size = inp.size(); |
| 152 | |
| 153 | if (inp_size < LLAMA_NGRAM_STATIC) { |
| 154 | return; |
| 155 | } |
| 156 | |
| 157 | while ((int) draft.size()-1 < n_draft) { |
| 158 | llama_token drafted_token = LLAMA_TOKEN_NULL; |
| 159 | |
| 160 | const int ngram_start_static = inp_size-LLAMA_NGRAM_STATIC + draft.size()-1; |
| 161 | common_ngram ngram_static; |
| 162 | for (int j = ngram_start_static; j < ngram_start_static + LLAMA_NGRAM_STATIC; ++j) { |
| 163 | ngram_static.tokens[j-ngram_start_static] = get_token(inp, draft, j); |
| 164 | } |
| 165 | common_ngram_cache::iterator part_static_it = nc_static.find(ngram_static); |
| 166 | common_ngram_cache_part part_static; |
| 167 | if (part_static_it != nc_static.end()) { |
| 168 | part_static = part_static_it->second; |
| 169 | } |
| 170 | |
| 171 | // cd = context + dynamic |
| 172 | std::vector<common_ngram> ngrams_cd; |
| 173 | for (int ngram_size_cd = ngram_min; ngram_size_cd <= ngram_max; ++ngram_size_cd) { |
| 174 | const int ngram_start_cd = inp_size-ngram_size_cd + draft.size()-1; |
| 175 | common_ngram ngram_cd; |
| 176 | for (int j = ngram_start_cd; j < ngram_start_cd + ngram_size_cd; ++j) { |
| 177 | ngram_cd.tokens[j-ngram_start_cd] = get_token(inp, draft, j); |
| 178 | } |
| 179 | ngrams_cd.push_back(ngram_cd); |
| 180 | } |
| 181 | if (drafted_token == LLAMA_TOKEN_NULL) { |
| 182 | drafted_token = try_draft(nc_context, ngrams_cd, part_static, draft_min_sample_size_lax, draft_min_percent_lax); |
| 183 | } |
| 184 | if (drafted_token == LLAMA_TOKEN_NULL) { |
| 185 | drafted_token = try_draft(nc_dynamic, ngrams_cd, part_static, draft_min_sample_size_strict, draft_min_percent_strict); |
| 186 | } |
| 187 | if (drafted_token == LLAMA_TOKEN_NULL) { |
| 188 | drafted_token = try_draft(nc_static, ngram_static); |
| 189 | } |
| 190 | |
| 191 | if (drafted_token == LLAMA_TOKEN_NULL) { |
| 192 | break; |
| 193 | } |
| 194 | |
| 195 | LOG(" - draft candidate: token=%d\n", drafted_token); |
| 196 | draft.push_back(drafted_token); |
| 197 | } |
| 198 | } |
| 199 | |
| 200 | void common_ngram_cache_save(common_ngram_cache & ngram_cache, std::string & filename) { |
| 201 | std::ofstream file_out(filename, std::ios::binary); |