| 234 | } |
| 235 | |
| 236 | std::vector<unsigned char> CoreBPE::decode_bytes(const std::vector<int>& tokens) const { |
| 237 | std::vector<unsigned char> ret; |
| 238 | ret.reserve(tokens.size() * 2); |
| 239 | for (int token : tokens) { |
| 240 | std::vector<unsigned char> token_bytes; |
| 241 | auto it = decoder.find(token); |
| 242 | if (it != decoder.end()) { |
| 243 | token_bytes = it->second; |
| 244 | } else { |
| 245 | auto special_it = special_tokens_decoder.find(token); |
| 246 | if (special_it != special_tokens_decoder.end()) { |
| 247 | token_bytes = special_it->second; |
| 248 | } else { |
| 249 | throw TiktokenError("Invalid token for decoding: " + std::to_string(token)); |
| 250 | } |
| 251 | } |
| 252 | ret.insert(ret.end(), token_bytes.begin(), token_bytes.end()); |
| 253 | } |
| 254 | return ret; |
| 255 | } |
| 256 | |
| 257 | |
| 258 | std::vector<std::string> CoreBPE::special_tokens() const { |