| 634 | } |
| 635 | } |
| 636 | |
| 637 | // Detect pre-tokenizer type. |
| 638 | int pre_key = gguf_find_key(gctx, "tokenizer.ggml.pre"); |
| 639 | if (pre_key >= 0) { |
| 640 | const char * pre = gguf_get_val_str(gctx, pre_key); |
| 641 | if (pre && std::strcmp(pre, "qwen35") == 0) { |
| 642 | pre_type_ = PreTokenizer::QWEN35; |
| 643 | } else { |
| 644 | pre_type_ = PreTokenizer::QWEN2; |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | // Load special token IDs. |
| 649 | auto get_i32 = [&](const char * key) -> int32_t { |
| 650 | int k = gguf_find_key(gctx, key); |
| 651 | if (k < 0) return -1; |
| 652 | return (int32_t)gguf_get_val_u32(gctx, k); |
| 653 | }; |
| 654 | |
| 655 | bos_id_ = get_i32("tokenizer.ggml.bos_token_id"); |
| 656 | eos_id_ = get_i32("tokenizer.ggml.eos_token_id"); |
| 657 | eos_chat_id_ = get_i32("tokenizer.ggml.eot_token_id"); |
| 658 | if (eos_chat_id_ < 0) { |
| 659 | // Qwen3 uses <|im_end|> as EOT. |
| 660 | auto eot = token_to_id_.find("<|im_end|>"); |
| 661 | if (eot != token_to_id_.end()) eos_chat_id_ = eot->second; |
| 662 | } |
| 663 | if (eos_chat_id_ < 0) { |
| 664 | // Gemma4 uses <turn|> as end-of-turn. |
| 665 | auto eot = token_to_id_.find("<turn|>"); |
| 666 | if (eot != token_to_id_.end()) eos_chat_id_ = eot->second; |
| 667 | } |
| 668 | |
| 669 | gguf_free(gctx); |
| 670 | |
| 671 | std::fprintf(stderr, "[tokenizer] loaded vocab=%d merges=%zu bos=%d eos=%d eot=%d pre=%s sp=%s\n", |
| 672 | n_vocab, merge_rank_.size(), bos_id_, eos_id_, eos_chat_id_, |
| 673 | pre_type_ == PreTokenizer::QWEN35 ? "qwen35" : "qwen2", |
| 674 | is_sentencepiece_ ? "yes" : "no"); |
| 675 | return true; |
| 676 | } |
| 677 | |
| 678 | std::vector<int32_t> Tokenizer::encode(const std::string & text) const { |
| 679 | // If no added tokens, fast path: pre-tokenize → BPE entire text. |
| 680 | if (added_tokens_.empty()) { |
| 681 | std::vector<std::string> pieces = pre_tokenize(text); |
| 682 | std::vector<int32_t> ids; |
| 683 | for (const auto & piece : pieces) { |
| 684 | auto piece_ids = bpe_encode_piece(piece); |
| 685 | ids.insert(ids.end(), piece_ids.begin(), piece_ids.end()); |
| 686 | } |
| 687 | return ids; |
| 688 | } |