| 221 | } |
| 222 | |
| 223 | common_ngram_cache common_ngram_cache_load(std::string & filename) { |
| 224 | std::ifstream hashmap_file(filename, std::ios::binary); |
| 225 | if (!hashmap_file) { |
| 226 | throw std::ifstream::failure("Unable to open file " + filename); |
| 227 | } |
| 228 | common_ngram_cache ngram_cache; |
| 229 | |
| 230 | common_ngram ngram; |
| 231 | int32_t ntokens; |
| 232 | llama_token token; |
| 233 | int32_t count; |
| 234 | |
| 235 | char * ngramc = reinterpret_cast<char*>(&ngram); |
| 236 | char * ntokensc = reinterpret_cast<char*>(&ntokens); |
| 237 | char * tokenc = reinterpret_cast<char*>(&token); |
| 238 | char * countc = reinterpret_cast<char*>(&count); |
| 239 | while(hashmap_file.read(ngramc, sizeof(common_ngram))) { |
| 240 | GGML_ASSERT(!hashmap_file.eof()); |
| 241 | GGML_ASSERT(hashmap_file.read(ntokensc, sizeof(int32_t))); |
| 242 | GGML_ASSERT(ntokens > 0); |
| 243 | common_ngram_cache_part token_counts; |
| 244 | |
| 245 | for (int i = 0; i < ntokens; ++i) { |
| 246 | GGML_ASSERT(!hashmap_file.eof()); |
| 247 | GGML_ASSERT(hashmap_file.read(tokenc, sizeof(llama_token))); |
| 248 | GGML_ASSERT(!hashmap_file.eof()); |
| 249 | GGML_ASSERT(hashmap_file.read(countc, sizeof(int32_t))); |
| 250 | GGML_ASSERT(count > 0); |
| 251 | token_counts.emplace(token, count); |
| 252 | } |
| 253 | |
| 254 | ngram_cache.emplace(ngram, token_counts); |
| 255 | } |
| 256 | GGML_ASSERT(hashmap_file.eof()); |
| 257 | |
| 258 | return ngram_cache; |
| 259 | } |
| 260 | |
| 261 | void common_ngram_cache_merge(common_ngram_cache & ngram_cache_target, common_ngram_cache & ngram_cache_add) { |
| 262 | for (std::pair<common_ngram, common_ngram_cache_part> ngram_part : ngram_cache_add) { |