| 585 | } |
| 586 | |
| 587 | std::string decode_bpe(const BpeVocabulary & vocab, const std::vector<int32_t> & token_ids, bool skip_special_tokens) { |
| 588 | std::string decoded; |
| 589 | for (const int32_t token_id : token_ids) { |
| 590 | const auto & token = vocab.require_token_data(token_id); |
| 591 | if (skip_special_tokens && (token.attr & (TOKEN_ATTR_CONTROL | TOKEN_ATTR_USER_DEFINED | TOKEN_ATTR_UNKNOWN))) { |
| 592 | continue; |
| 593 | } |
| 594 | if (token.attr & (TOKEN_ATTR_CONTROL | TOKEN_ATTR_USER_DEFINED | TOKEN_ATTR_UNKNOWN)) { |
| 595 | decoded += token.text; |
| 596 | continue; |
| 597 | } |
| 598 | |
| 599 | const auto cpts = unicode_cpts_from_utf8(token.text); |
| 600 | for (const auto cpt : cpts) { |
| 601 | const auto utf8 = unicode_cpt_to_utf8(cpt); |
| 602 | try { |
| 603 | decoded.push_back(static_cast<char>(unicode_utf8_to_byte(utf8))); |
| 604 | } catch (const std::out_of_range &) { |
| 605 | decoded += utf8; |
| 606 | } |
| 607 | } |
| 608 | } |
| 609 | return decoded; |
| 610 | } |
| 611 | |
| 612 | } // namespace llama_tokenizer_vendor |
no test coverage detected