| 200 | } |
| 201 | |
| 202 | static inline std::wstring unicode_wstring_from_utf8(const std::string & s) { |
| 203 | #if defined(__clang__) |
| 204 | // disable C++17 deprecation warning for std::codecvt_utf8 |
| 205 | # pragma clang diagnostic push |
| 206 | # pragma clang diagnostic ignored "-Wdeprecated-declarations" |
| 207 | #elif defined(__GNUC__) |
| 208 | # pragma GCC diagnostic push |
| 209 | # pragma GCC diagnostic ignored "-Wdeprecated-declarations" |
| 210 | #endif |
| 211 | |
| 212 | std::wstring_convert<std::codecvt_utf8<wchar_t>> conv; |
| 213 | |
| 214 | #if defined(__clang__) |
| 215 | # pragma clang diagnostic pop |
| 216 | #elif defined(__GNUC__) |
| 217 | # pragma GCC diagnostic pop |
| 218 | #endif |
| 219 | |
| 220 | return conv.from_bytes(s); |
| 221 | } |
| 222 | |
| 223 | static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) { |
| 224 | std::vector<std::string> bpe_encoded_words; |
no outgoing calls
no test coverage detected