| 198 | } |
| 199 | |
| 200 | void common_ngram_cache_save(common_ngram_cache & ngram_cache, const std::string & filename) { |
| 201 | std::ofstream file_out(filename, std::ios::binary); |
| 202 | for (std::pair<common_ngram, common_ngram_cache_part> item : ngram_cache) { |
| 203 | const common_ngram ngram = item.first; |
| 204 | common_ngram_cache_part token_counts = item.second; |
| 205 | GGML_ASSERT(!token_counts.empty()); |
| 206 | const int32_t ntokens = token_counts.size(); |
| 207 | GGML_ASSERT(ntokens > 0); |
| 208 | |
| 209 | file_out.write(reinterpret_cast<const char *>(&ngram), sizeof(common_ngram)); |
| 210 | file_out.write(reinterpret_cast<const char *>(&ntokens), sizeof(int32_t)); |
| 211 | for (std::pair<llama_token, int32_t> item2 : token_counts) { |
| 212 | const llama_token token = item2.first; |
| 213 | const int32_t count = item2.second; |
| 214 | GGML_ASSERT(count > 0); |
| 215 | |
| 216 | file_out.write(reinterpret_cast<const char *>(&token), sizeof(llama_token)); |
| 217 | file_out.write(reinterpret_cast<const char *>(&count), sizeof(int32_t)); |
| 218 | } |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | common_ngram_cache common_ngram_cache_load(const std::string & filename) { |
| 223 | std::ifstream hashmap_file(filename, std::ios::binary); |