| 2870 | } |
| 2871 | |
| 2872 | static std::string llama_decode_text(const std::string & text) { |
| 2873 | std::string decoded_text; |
| 2874 | |
| 2875 | const auto cpts = unicode_cpts_from_utf8(text); |
| 2876 | for (const auto cpt : cpts) { |
| 2877 | const auto utf8 = unicode_cpt_to_utf8(cpt); |
| 2878 | try { |
| 2879 | decoded_text += unicode_utf8_to_byte(utf8); |
| 2880 | } catch (const std::out_of_range & /*e*/) { |
| 2881 | decoded_text += "[UNK_BYTE_0x"; |
| 2882 | for (const auto c : utf8) { |
| 2883 | decoded_text += format("%02x", (uint8_t) c); |
| 2884 | } |
| 2885 | decoded_text += text + "]"; |
| 2886 | } |
| 2887 | } |
| 2888 | |
| 2889 | return decoded_text; |
| 2890 | } |
| 2891 | |
| 2892 | std::vector<llama_token> llama_vocab::impl::tokenize( |
| 2893 | const std::string & raw_text, |
no test coverage detected