| 244 | } |
| 245 | |
| 246 | void gpt_split_words(std::string str, std::vector<std::string>& words) { |
| 247 | const std::string pattern = R"('s|'t|'re|'ve|'m|'ll|'d| ?[[:alpha:]]+| ?[[:digit:]]+| ?[^\s[:alpha:][:digit:]]+|\s+(?!\S)|\s+)"; |
| 248 | const std::regex re(pattern); |
| 249 | std::smatch m; |
| 250 | |
| 251 | while (std::regex_search(str, m, re)) { |
| 252 | for (auto x : m) { |
| 253 | words.push_back(x); |
| 254 | } |
| 255 | str = m.suffix(); |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | std::vector<gpt_vocab::id> gpt_tokenize(const gpt_vocab & vocab, const std::string & text) { |
| 260 | std::vector<std::string> words; |