| 257 | } |
| 258 | |
| 259 | std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) { |
| 260 | std::vector<std::string> words; |
| 261 | |
| 262 | // first split the text into words |
| 263 | { |
| 264 | std::string str = text; |
| 265 | |
| 266 | // Generate the subpattern from the special_tokens vector if it's not empty |
| 267 | if (!vocab.special_tokens.empty()) { |
| 268 | const std::regex escape(R"([\[\\\^\$\.\|\?\*\+\(\)\{\}])"); |
| 269 | std::string special_tokens_subpattern; |
| 270 | for (const auto & token : vocab.special_tokens) { |
| 271 | if (!special_tokens_subpattern.empty()) { |
| 272 | special_tokens_subpattern += "|"; |
| 273 | } |
| 274 | special_tokens_subpattern += std::regex_replace(token, escape, R"(\$&)"); |
| 275 | } |
| 276 | |
| 277 | std::regex re(special_tokens_subpattern); |
| 278 | std::smatch m; |
| 279 | // Split the text by special tokens. |
| 280 | while (std::regex_search(str, m, re)) { |
| 281 | // Split the substrings in-between special tokens into words. |
| 282 | gpt_split_words(m.prefix(), words); |
| 283 | // Add matched special tokens as words. |
| 284 | for (auto x : m) { |
| 285 | words.push_back(x); |
| 286 | } |
| 287 | str = m.suffix(); |
| 288 | } |
| 289 | // Remaining text without special tokens will be handled below. |
| 290 | } |
| 291 | |
| 292 | gpt_split_words(str, words); |
| 293 | } |
| 294 | |
| 295 | // find the longest token that forms each word in words: |
| 296 | std::vector<gpt_vocab::id> tokens; |
| 297 | for (const auto & word : words) { |
| 298 | for (int i = 0; i < (int) word.size(); ){ |
| 299 | for (int j = word.size() - 1; j >= i; j--){ |
| 300 | auto cand = word.substr(i, j-i+1); |
| 301 | auto it = vocab.token_to_id.find(cand); |
| 302 | if (it != vocab.token_to_id.end()){ // word.substr(i, j-i+1) in vocab |
| 303 | tokens.push_back(it->second); |
| 304 | i = j + 1; |
| 305 | break; |
| 306 | } |
| 307 | else if (j == i){ // word.substr(i, 1) has no matching |
| 308 | fprintf(stderr, "%s: unknown token '%s'\n", __func__, word.substr(i, 1).data()); |
| 309 | i++; |
| 310 | } |
| 311 | } |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | return tokens; |
| 316 | } |