| 170 | } |
| 171 | |
| 172 | static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() { |
| 173 | std::unordered_map<std::string, uint8_t> map; |
| 174 | for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~' |
| 175 | assert(0 <= ch && ch < 256); |
| 176 | map[unicode_cpt_to_utf8(ch)] = ch; |
| 177 | } |
| 178 | for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬' |
| 179 | assert(0 <= ch && ch < 256); |
| 180 | map[unicode_cpt_to_utf8(ch)] = ch; |
| 181 | } |
| 182 | for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ' |
| 183 | assert(0 <= ch && ch < 256); |
| 184 | map[unicode_cpt_to_utf8(ch)] = ch; |
| 185 | } |
| 186 | auto n = 0; |
| 187 | for (int ch = 0; ch < 256; ++ch) { |
| 188 | if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) { |
| 189 | map[unicode_cpt_to_utf8(256 + n)] = ch; |
| 190 | ++n; |
| 191 | } |
| 192 | } |
| 193 | return map; |
| 194 | } |
| 195 | |
| 196 | static std::vector<std::string> unicode_byte_encoding_process(const std::vector<std::string> & bpe_words) { |
| 197 | std::vector<std::string> bpe_encoded_words; |
no test coverage detected