| 340 | } |
| 341 | |
| 342 | void llm_tokenizer_bpe_session::tokenize(const std::string & text, std::vector<int32_t> & output) { |
| 343 | int final_prev_index = -1; |
| 344 | const auto word_collection = unicode_regex_split(text, tokenizer.regex_exprs, tokenizer.byte_encode); |
| 345 | |
| 346 | symbols_final.clear(); |
| 347 | for (const auto & word : word_collection) { |
| 348 | work_queue = llm_bigram_bpe::queue(); |
| 349 | symbols.clear(); |
| 350 | |
| 351 | int index = 0; |
| 352 | size_t offset = 0; |
| 353 | while (offset < word.size()) { |
| 354 | llm_symbol sym; |
| 355 | const size_t char_len = std::min(word.size() - offset, static_cast<size_t>(unicode_len_utf8(word[offset]))); |
| 356 | sym.text = word.c_str() + offset; |
| 357 | sym.n = char_len; |
| 358 | offset += sym.n; |
| 359 | sym.prev = index - 1; |
| 360 | sym.next = offset == word.size() ? -1 : index + 1; |
| 361 | index++; |
| 362 | symbols.emplace_back(sym); |
| 363 | } |
| 364 | |
| 365 | for (int i = 1; i < static_cast<int>(symbols.size()); ++i) { |
| 366 | add_new_bigram(i - 1, i); |
| 367 | } |
| 368 | |
| 369 | while (!work_queue.empty()) { |
| 370 | auto bigram = work_queue.pop_move(); |
| 371 | |
| 372 | auto & left_symbol = symbols[bigram.left]; |
| 373 | auto & right_symbol = symbols[bigram.right]; |
| 374 | |
| 375 | if (left_symbol.n == 0 || right_symbol.n == 0) { |
| 376 | continue; |
| 377 | } |
| 378 | const std::string left_token(left_symbol.text, left_symbol.n); |
| 379 | const std::string right_token(right_symbol.text, right_symbol.n); |
| 380 | if (left_token + right_token != bigram.text) { |
| 381 | continue; |
| 382 | } |
| 383 | |
| 384 | left_symbol.n += right_symbol.n; |
| 385 | right_symbol.n = 0; |
| 386 | left_symbol.next = right_symbol.next; |
| 387 | if (right_symbol.next >= 0) { |
| 388 | symbols[right_symbol.next].prev = bigram.left; |
| 389 | } |
| 390 | |
| 391 | add_new_bigram(left_symbol.prev, bigram.left); |
| 392 | add_new_bigram(bigram.left, left_symbol.next); |
| 393 | } |
| 394 | |
| 395 | for (auto & sym : symbols) { |
| 396 | if (sym.n > 0) { |
| 397 | sym.prev = final_prev_index; |
| 398 | sym.next = -1; |
| 399 | if (final_prev_index != -1) { |