MCPcopy Create free account
hub / github.com/appdevforall/CodeOnTheGo / unicode_utf8_to_byte_map

Function unicode_utf8_to_byte_map

subprojects/llama.cpp/src/unicode.cpp:178–200  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

176}
177
178static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
179 std::unordered_map<std::string, uint8_t> map;
180 for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
181 assert(0 <= ch && ch < 256);
182 map[unicode_cpt_to_utf8(ch)] = ch;
183 }
184 for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
185 assert(0 <= ch && ch < 256);
186 map[unicode_cpt_to_utf8(ch)] = ch;
187 }
188 for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
189 assert(0 <= ch && ch < 256);
190 map[unicode_cpt_to_utf8(ch)] = ch;
191 }
192 auto n = 0;
193 for (int ch = 0; ch < 256; ++ch) {
194 if (map.find(unicode_cpt_to_utf8(ch)) == map.end()) {
195 map[unicode_cpt_to_utf8(256 + n)] = ch;
196 ++n;
197 }
198 }
199 return map;
200}
201
202static inline std::wstring unicode_wstring_from_utf8(const std::string & s) {
203#if defined(__clang__)

Callers 1

unicode_utf8_to_byteFunction · 0.85

Calls 3

unicode_cpt_to_utf8Function · 0.85
findMethod · 0.65
endMethod · 0.45

Tested by

no test coverage detected