| 200 | } |
| 201 | |
| 202 | static inline std::wstring unicode_wstring_from_utf8(const std::string & s) { |
| 203 | #if defined(__clang__) |
| 204 | // disable C++17 deprecation warning for std::codecvt_utf8 |
| 205 | # pragma clang diagnostic push |
| 206 | # pragma clang diagnostic ignored "-Wdeprecated-declarations" |
| 207 | #endif |
| 208 | |
| 209 | std::wstring_convert<std::codecvt_utf8<wchar_t>> conv; |
| 210 | |
| 211 | #if defined(__clang__) |
| 212 | # pragma clang diagnostic pop |
| 213 | #endif |
| 214 | |
| 215 | return conv.from_bytes(s); |
| 216 | } |
| 217 | |
| 218 | static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) { |
| 219 | std::vector<std::string> bpe_encoded_words; |
no outgoing calls
no test coverage detected