| 78 | NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MistralTokenizer, config, vocab); |
| 79 | |
| 80 | static std::string base64_decode(const std::string &in) { |
| 81 | std::string out; |
| 82 | std::vector<int> T(256,-1); |
| 83 | for (int i=0; i<64; i++) T["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[i]] = i; |
| 84 | |
| 85 | int val=0, valb=-8; |
| 86 | for (unsigned char c : in) { |
| 87 | if (T[c] == -1) break; |
| 88 | val = (val << 6) + T[c]; |
| 89 | valb += 6; |
| 90 | if (valb >= 0) { |
| 91 | out.push_back(char((val>>valb)&0xFF)); |
| 92 | valb -= 8; |
| 93 | } |
| 94 | } |
| 95 | return out; |
| 96 | } |
| 97 | |
| 98 | |
| 99 | void LoadTokenizer(const std::string& filename, MistralTokenizer& tokenizer) { |