| 2647 | } |
| 2648 | |
| 2649 | int32_t llama_vocab::impl::detokenize( |
| 2650 | const llama_token * tokens, |
| 2651 | int32_t n_tokens, |
| 2652 | char * text, |
| 2653 | int32_t text_len_max, |
| 2654 | bool remove_special, |
| 2655 | bool unparse_special) const { |
| 2656 | if (type == LLAMA_VOCAB_TYPE_NONE) { |
| 2657 | return 0; |
| 2658 | } |
| 2659 | |
| 2660 | GGML_ASSERT(tokenizer && "Tokenizer not initialized. Call llama_vocab::init_tokenizer() first."); |
| 2661 | |
| 2662 | int32_t avail = text_len_max; |
| 2663 | int32_t total = 0; |
| 2664 | |
| 2665 | // remove the leading space |
| 2666 | bool remove_space = add_space_prefix; |
| 2667 | |
| 2668 | if (remove_special && add_bos) { |
| 2669 | if (n_tokens > 0 && tokens[0] == special_bos_id) { |
| 2670 | remove_space = false; |
| 2671 | n_tokens--; |
| 2672 | tokens++; |
| 2673 | } |
| 2674 | } |
| 2675 | |
| 2676 | if (remove_special && add_eos) { |
| 2677 | if (n_tokens > 0 && tokens[n_tokens - 1] == special_eos_id) { |
| 2678 | n_tokens--; |
| 2679 | } |
| 2680 | } |
| 2681 | |
| 2682 | for (int32_t i = 0; i < n_tokens; ++i) { |
| 2683 | GGML_ASSERT(avail >= 0); |
| 2684 | int32_t n_chars = token_to_piece(tokens[i], text, avail, remove_space, unparse_special); |
| 2685 | remove_space = false; |
| 2686 | if (n_chars < 0) { |
| 2687 | avail = 0; |
| 2688 | total -= n_chars; |
| 2689 | } else if (n_chars > 0) { |
| 2690 | avail -= n_chars; |
| 2691 | text += n_chars; |
| 2692 | total += n_chars; |
| 2693 | } |
| 2694 | } |
| 2695 | |
| 2696 | if (total > text_len_max) { |
| 2697 | return -total; |
| 2698 | } |
| 2699 | |
| 2700 | if (clean_spaces) { |
| 2701 | text -= total; // restart text |
| 2702 | |
| 2703 | // first pass: characters ?!., //TODO: where do these characters come from? |
| 2704 | const int32_t total1 = total; |
| 2705 | total = total ? 1 : 0; |
| 2706 | for (int32_t i = 1; i < total1; ++i) { |
no test coverage detected