| 634 | } |
| 635 | |
| 636 | std::vector<int32_t> Tokenizer::encode(const std::string & text) const { |
| 637 | // If no added tokens, fast path: pre-tokenize → BPE entire text. |
| 638 | if (added_tokens_.empty()) { |
| 639 | std::vector<std::string> pieces = pre_tokenize(text); |
| 640 | std::vector<int32_t> ids; |
| 641 | for (const auto & piece : pieces) { |
| 642 | auto piece_ids = bpe_encode_piece(piece); |
| 643 | ids.insert(ids.end(), piece_ids.begin(), piece_ids.end()); |
| 644 | } |
| 645 | return ids; |
| 646 | } |
| 647 | |
| 648 | // Split text into segments: alternating normal text and special tokens. |
| 649 | // Special tokens are matched greedily (longest first). |
| 650 | std::vector<int32_t> ids; |
| 651 | size_t pos = 0; |
| 652 | while (pos < text.size()) { |
| 653 | // Try to match any added token at current position. |
| 654 | bool matched = false; |
| 655 | for (const auto & [tok_str, tok_id] : added_tokens_) { |
| 656 | if (pos + tok_str.size() <= text.size() && |
| 657 | text.compare(pos, tok_str.size(), tok_str) == 0) { |
| 658 | ids.push_back(tok_id); |
| 659 | pos += tok_str.size(); |
| 660 | matched = true; |
| 661 | break; |
| 662 | } |
| 663 | } |
| 664 | if (matched) continue; |
| 665 | |
| 666 | // Find the next special token (or end of string). |
| 667 | size_t next_special = text.size(); |
| 668 | for (const auto & [tok_str, tok_id] : added_tokens_) { |
| 669 | size_t found = text.find(tok_str, pos); |
| 670 | if (found != std::string::npos && found < next_special) { |
| 671 | next_special = found; |
| 672 | } |
| 673 | } |
| 674 | |
| 675 | // Pre-tokenize + BPE the normal segment. |
| 676 | std::string segment = text.substr(pos, next_special - pos); |
| 677 | std::vector<std::string> pieces = pre_tokenize(segment); |
| 678 | for (const auto & piece : pieces) { |
| 679 | auto piece_ids = bpe_encode_piece(piece); |
| 680 | ids.insert(ids.end(), piece_ids.begin(), piece_ids.end()); |
| 681 | } |
| 682 | pos = next_special; |
| 683 | } |
| 684 | return ids; |
| 685 | } |
| 686 | |
| 687 | // GPT-2 byte-level BPE uses a Unicode mapping where each byte 0-255 is |
| 688 | // represented by a specific Unicode codepoint. Bytes that already have a |