MCPcopy Create free account
hub / github.com/appdevforall/CodeOnTheGo / unicode_byte_to_utf8_map

Function unicode_byte_to_utf8_map

subprojects/llama.cpp/src/unicode.cpp:154–176  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

152}
153
154static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {
155 std::unordered_map<uint8_t, std::string> map;
156 for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
157 assert(0 <= ch && ch < 256);
158 map[ch] = unicode_cpt_to_utf8(ch);
159 }
160 for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
161 assert(0 <= ch && ch < 256);
162 map[ch] = unicode_cpt_to_utf8(ch);
163 }
164 for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
165 assert(0 <= ch && ch < 256);
166 map[ch] = unicode_cpt_to_utf8(ch);
167 }
168 auto n = 0;
169 for (int ch = 0; ch < 256; ++ch) {
170 if (map.find(ch) == map.end()) {
171 map[ch] = unicode_cpt_to_utf8(256 + n);
172 ++n;
173 }
174 }
175 return map;
176}
177
178static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
179 std::unordered_map<std::string, uint8_t> map;

Callers 1

unicode_byte_to_utf8Function · 0.85

Calls 3

unicode_cpt_to_utf8Function · 0.85
findMethod · 0.65
endMethod · 0.45

Tested by

no test coverage detected