| 7658 | } |
| 7659 | |
| 7660 | static std::vector<llama_vocab::id> llama_tokenize_internal(const llama_vocab & vocab, std::string raw_text, bool bos, bool special) { |
| 7661 | std::vector<llama_vocab::id> output; |
| 7662 | |
| 7663 | // OG tokenizer behavior: |
| 7664 | // |
| 7665 | // tokenizer.encode('', add_bos=True) returns [1] |
| 7666 | // tokenizer.encode('', add_bos=False) returns [] |
| 7667 | |
| 7668 | if (bos && vocab.special_bos_id != -1) { |
| 7669 | output.push_back(vocab.special_bos_id); |
| 7670 | } |
| 7671 | |
| 7672 | if (raw_text.empty()) { |
| 7673 | return output; |
| 7674 | } |
| 7675 | |
| 7676 | std::forward_list<fragment_buffer_variant> fragment_buffer; |
| 7677 | fragment_buffer.emplace_front( raw_text, 0, raw_text.length() ); |
| 7678 | |
| 7679 | if (special) tokenizer_st_partition( vocab, fragment_buffer ); |
| 7680 | |
| 7681 | switch (vocab.type) { |
| 7682 | case LLAMA_VOCAB_TYPE_SPM: |
| 7683 | { |
| 7684 | for (const auto & fragment: fragment_buffer) |
| 7685 | { |
| 7686 | if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) |
| 7687 | { |
| 7688 | // without adding this leading whitespace, we do not get the same results as the original tokenizer |
| 7689 | |
| 7690 | // TODO: It's likely possible to get rid of this string copy entirely |
| 7691 | // by modifying llm_tokenizer_x to operate with string offsets like pre-tokenizer |
| 7692 | // and passing 'add space prefix' as bool argument |
| 7693 | // |
| 7694 | auto raw_text = (special ? "" : " ") + fragment.raw_text.substr(fragment.offset, fragment.length); |
| 7695 | |
| 7696 | #ifdef PRETOKENIZERDEBUG |
| 7697 | fprintf(stderr,"TT: (%ld %ld %ld) '%s'\n", raw_text.length(), fragment.offset, fragment.length, raw_text.c_str()); |
| 7698 | #endif |
| 7699 | llm_tokenizer_spm tokenizer(vocab); |
| 7700 | llama_escape_whitespace(raw_text); |
| 7701 | tokenizer.tokenize(raw_text, output); |
| 7702 | } |
| 7703 | else // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) |
| 7704 | { |
| 7705 | output.push_back(fragment.token); |
| 7706 | } |
| 7707 | } |
| 7708 | } break; |
| 7709 | case LLAMA_VOCAB_TYPE_BPE: |
| 7710 | { |
| 7711 | for (const auto & fragment: fragment_buffer) |
| 7712 | { |
| 7713 | if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) |
| 7714 | { |
| 7715 | auto raw_text = fragment.raw_text.substr(fragment.offset, fragment.length); |
| 7716 | |
| 7717 | #ifdef PRETOKENIZERDEBUG |
no test coverage detected