| 3193 | } |
| 3194 | |
| 3195 | int32_t llama_vocab::impl::detokenize( |
| 3196 | const llama_token * tokens, |
| 3197 | int32_t n_tokens, |
| 3198 | char * text, |
| 3199 | int32_t text_len_max, |
| 3200 | bool remove_special, |
| 3201 | bool unparse_special) const { |
| 3202 | if (type == LLAMA_VOCAB_TYPE_NONE) { |
| 3203 | return 0; |
| 3204 | } |
| 3205 | |
| 3206 | GGML_ASSERT(tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first."); |
| 3207 | |
| 3208 | int32_t avail = text_len_max; |
| 3209 | int32_t total = 0; |
| 3210 | |
| 3211 | // remove the leading space |
| 3212 | bool remove_space = add_space_prefix; |
| 3213 | |
| 3214 | if (remove_special && add_bos) { |
| 3215 | if (n_tokens > 0 && tokens[0] == special_bos_id) { |
| 3216 | remove_space = false; |
| 3217 | n_tokens--; |
| 3218 | tokens++; |
| 3219 | } |
| 3220 | } |
| 3221 | |
| 3222 | if (remove_special && add_eos) { |
| 3223 | if (n_tokens > 0 && tokens[n_tokens - 1] == special_eos_id) { |
| 3224 | n_tokens--; |
| 3225 | } |
| 3226 | } |
| 3227 | |
| 3228 | for (int32_t i = 0; i < n_tokens; ++i) { |
| 3229 | GGML_ASSERT(avail >= 0); |
| 3230 | int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special); |
| 3231 | remove_space = false; |
| 3232 | if (n_chars < 0) { |
| 3233 | avail = 0; |
| 3234 | total -= n_chars; |
| 3235 | } else if (n_chars > 0) { |
| 3236 | avail -= n_chars; |
| 3237 | text += n_chars; |
| 3238 | total += n_chars; |
| 3239 | } |
| 3240 | } |
| 3241 | |
| 3242 | if (total > text_len_max) { |
| 3243 | return -total; |
| 3244 | } |
| 3245 | |
| 3246 | if (clean_spaces) { |
| 3247 | text -= total; // restart text |
| 3248 | |
| 3249 | // first pass: characters ?!., //TODO: where do these characters come from? |
| 3250 | const int32_t total1 = total; |
| 3251 | total = total ? 1 : 0; |
| 3252 | for (int32_t i = 1; i < total1; ++i) { |
no test coverage detected