| 438 | } |
| 439 | |
| 440 | void tokenizer_st_partition( |
| 441 | const BpeVocabulary & vocab, |
| 442 | std::forward_list<fragment_buffer_variant> & buffer, |
| 443 | bool parse_special) { |
| 444 | for (const int32_t special_id : vocab.special_tokens_sorted) { |
| 445 | const auto & data = vocab.require_token_data(special_id); |
| 446 | const auto & text = data.text; |
| 447 | |
| 448 | if (!parse_special && (data.attr & (TOKEN_ATTR_CONTROL | TOKEN_ATTR_UNKNOWN))) { |
| 449 | continue; |
| 450 | } |
| 451 | |
| 452 | auto it = buffer.begin(); |
| 453 | while (it != buffer.end()) { |
| 454 | auto & fragment = *it; |
| 455 | if (fragment.type != FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { |
| 456 | ++it; |
| 457 | continue; |
| 458 | } |
| 459 | |
| 460 | const auto & raw_text = fragment.raw_text; |
| 461 | auto raw_text_base_offset = fragment.offset; |
| 462 | auto raw_text_base_length = fragment.length; |
| 463 | |
| 464 | while (true) { |
| 465 | const auto match = std::string_view(raw_text.data(), raw_text_base_offset + raw_text_base_length).find( |
| 466 | text, |
| 467 | raw_text_base_offset); |
| 468 | if (match == std::string::npos) { |
| 469 | break; |
| 470 | } |
| 471 | |
| 472 | const auto source = std::distance(buffer.begin(), it); |
| 473 | if (match > raw_text_base_offset) { |
| 474 | const int64_t left_reminder_offset = raw_text_base_offset; |
| 475 | int64_t left_reminder_length = static_cast<int64_t>(match - raw_text_base_offset); |
| 476 | if (data.attr & TOKEN_ATTR_LSTRIP) { |
| 477 | while (left_reminder_length > 0 && |
| 478 | std::isspace(static_cast<unsigned char>(raw_text[left_reminder_offset + left_reminder_length - 1]))) { |
| 479 | left_reminder_length--; |
| 480 | } |
| 481 | } |
| 482 | if (left_reminder_length > 0) { |
| 483 | buffer.emplace_after(it, raw_text, left_reminder_offset, left_reminder_length); |
| 484 | ++it; |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | buffer.emplace_after(it, special_id); |
| 489 | ++it; |
| 490 | |
| 491 | if (match + text.length() < raw_text_base_offset + raw_text_base_length) { |
| 492 | int64_t right_reminder_offset = static_cast<int64_t>(match + text.length()); |
| 493 | int64_t right_reminder_length = static_cast<int64_t>( |
| 494 | raw_text_base_length - ((match - raw_text_base_offset) + text.length())); |
| 495 | if (data.attr & TOKEN_ATTR_RSTRIP) { |
| 496 | while (right_reminder_length > 0 && |
| 497 | std::isspace(static_cast<unsigned char>(raw_text[right_reminder_offset]))) { |