| 120 | } |
| 121 | |
| 122 | void load_model_from_tokenizer_json(const engine::io::json::Value & tokenizer_json, BpeVocabulary & vocab) { |
| 123 | const auto & model = tokenizer_json.require("model"); |
| 124 | if (model.require("type").as_string() != "BPE") { |
| 125 | throw std::runtime_error("llama BPE tokenizer expects a BPE tokenizer.json model"); |
| 126 | } |
| 127 | for (const auto & [token, id] : model.require("vocab").as_object()) { |
| 128 | const int32_t token_id = static_cast<int32_t>(id.as_i64()); |
| 129 | vocab.token_to_id.emplace(token, token_id); |
| 130 | vocab.id_to_token.emplace(token_id, TokenData{token, 0}); |
| 131 | } |
| 132 | |
| 133 | int32_t rank = 0; |
| 134 | for (const auto & merge : model.require("merges").as_array()) { |
| 135 | if (merge.is_string()) { |
| 136 | add_merge(vocab, merge.as_string(), rank++); |
| 137 | continue; |
| 138 | } |
| 139 | const auto & pair = merge.as_array(); |
| 140 | if (pair.size() != 2 || !pair[0].is_string() || !pair[1].is_string()) { |
| 141 | throw std::runtime_error("tokenizer.json BPE merge must be a string or two-string array"); |
| 142 | } |
| 143 | vocab.bpe_ranks.emplace(pair_key(pair[0].as_string(), pair[1].as_string()), rank++); |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | void load_added_tokens_from_tokenizer_json(const engine::io::json::Value & tokenizer_json, BpeVocabulary & vocab) { |
| 148 | const auto * added = tokenizer_json.find("added_tokens"); |