| 68 | |
| 69 | |
| 70 | static std::vector<unsigned char> base64_decode(const std::string &in) { |
| 71 | std::vector<unsigned char> out; |
| 72 | std::vector<int> T(256,-1); |
| 73 | for (int i=0; i<64; i++) T["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[i]] = i; |
| 74 | |
| 75 | int val=0, valb=-8; |
| 76 | for (unsigned char c : in) { |
| 77 | if (T[c] == -1) break; |
| 78 | val = (val << 6) + T[c]; |
| 79 | valb += 6; |
| 80 | if (valb >= 0) { |
| 81 | out.push_back((val>>valb)&0xFF); |
| 82 | valb -= 8; |
| 83 | } |
| 84 | } |
| 85 | return out; |
| 86 | } |
| 87 | |
| 88 | |
| 89 | void LoadBPEFile(const std::string& filename, std::vector<VocabItem>& vocab) { |