| 2890 | } |
| 2891 | |
| 2892 | std::vector<llama_token> llama_vocab::impl::tokenize( |
| 2893 | const std::string & raw_text, |
| 2894 | bool add_special, |
| 2895 | bool parse_special) const { |
| 2896 | GGML_ASSERT(tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first."); |
| 2897 | |
| 2898 | std::vector<llama_token> output; |
| 2899 | std::forward_list<fragment_buffer_variant> fragment_buffer; |
| 2900 | |
| 2901 | if (!raw_text.empty()) { |
| 2902 | fragment_buffer.emplace_front(raw_text, 0, raw_text.length()); |
| 2903 | tokenizer_st_partition(fragment_buffer, parse_special); |
| 2904 | } |
| 2905 | |
| 2906 | switch (get_type()) { |
| 2907 | case LLAMA_VOCAB_TYPE_SPM: |
| 2908 | { |
| 2909 | // OG tokenizer behavior: |
| 2910 | // |
| 2911 | // tokenizer.encode('', add_special_tokens=True) returns [1] |
| 2912 | // tokenizer.encode('', add_special_tokens=False) returns [] |
| 2913 | |
| 2914 | bool is_prev_special = true; // prefix with space if first token |
| 2915 | |
| 2916 | if (add_special && add_bos) { |
| 2917 | GGML_ASSERT(special_bos_id != LLAMA_TOKEN_NULL); |
| 2918 | output.push_back(special_bos_id); |
| 2919 | is_prev_special = true; |
| 2920 | } |
| 2921 | |
| 2922 | for (const auto & fragment : fragment_buffer) { |
| 2923 | if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { |
| 2924 | std::string text; |
| 2925 | |
| 2926 | // prefix with space if previous is special |
| 2927 | if (add_space_prefix && is_prev_special) { |
| 2928 | text = ' '; |
| 2929 | } |
| 2930 | |
| 2931 | text += fragment.raw_text.substr(fragment.offset, fragment.length); |
| 2932 | |
| 2933 | #ifdef PRETOKENIZERDEBUG |
| 2934 | LLAMA_LOG_WARN("TT: (%ld %ld %ld) '%s'\n", text.length(), fragment.offset, fragment.length, text.c_str()); |
| 2935 | #endif |
| 2936 | llama_escape_whitespace(text); |
| 2937 | llm_tokenizer_spm_session session(vocab); |
| 2938 | session.tokenize(text, output); |
| 2939 | is_prev_special = false; |
| 2940 | } else { // if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) |
| 2941 | output.push_back(fragment.token); |
| 2942 | is_prev_special = true; |
| 2943 | } |
| 2944 | } |
| 2945 | |
| 2946 | if (add_special && add_bos && output.size() >= 2 && output[1] == special_bos_id) { |
| 2947 | LLAMA_LOG_WARN( |
| 2948 | "%s: Added a BOS token to the prompt as specified by the model but the prompt " |
| 2949 | "also starts with a BOS token. So now the final prompt starts with 2 BOS tokens. " |
no test coverage detected