| 220 | } |
| 221 | |
| 222 | common_ngram_cache common_ngram_cache_load(const std::string & filename) { |
| 223 | std::ifstream hashmap_file(filename, std::ios::binary); |
| 224 | if (!hashmap_file) { |
| 225 | throw std::ifstream::failure("Unable to open file " + filename); |
| 226 | } |
| 227 | common_ngram_cache ngram_cache; |
| 228 | |
| 229 | common_ngram ngram; |
| 230 | int32_t ntokens; |
| 231 | llama_token token; |
| 232 | int32_t count; |
| 233 | |
| 234 | char * ngramc = reinterpret_cast<char*>(&ngram); |
| 235 | char * ntokensc = reinterpret_cast<char*>(&ntokens); |
| 236 | char * tokenc = reinterpret_cast<char*>(&token); |
| 237 | char * countc = reinterpret_cast<char*>(&count); |
| 238 | while(hashmap_file.read(ngramc, sizeof(common_ngram))) { |
| 239 | GGML_ASSERT(!hashmap_file.eof()); |
| 240 | GGML_ASSERT(hashmap_file.read(ntokensc, sizeof(int32_t))); |
| 241 | GGML_ASSERT(ntokens > 0); |
| 242 | common_ngram_cache_part token_counts; |
| 243 | |
| 244 | for (int i = 0; i < ntokens; ++i) { |
| 245 | GGML_ASSERT(!hashmap_file.eof()); |
| 246 | GGML_ASSERT(hashmap_file.read(tokenc, sizeof(llama_token))); |
| 247 | GGML_ASSERT(!hashmap_file.eof()); |
| 248 | GGML_ASSERT(hashmap_file.read(countc, sizeof(int32_t))); |
| 249 | GGML_ASSERT(count > 0); |
| 250 | token_counts.emplace(token, count); |
| 251 | } |
| 252 | |
| 253 | ngram_cache.emplace(ngram, token_counts); |
| 254 | } |
| 255 | GGML_ASSERT(hashmap_file.eof()); |
| 256 | |
| 257 | return ngram_cache; |
| 258 | } |
| 259 | |
| 260 | void common_ngram_cache_merge(common_ngram_cache & ngram_cache_target, common_ngram_cache & ngram_cache_add) { |
| 261 | for (std::pair<common_ngram, common_ngram_cache_part> ngram_part : ngram_cache_add) { |