MCPcopy Create free account
hub / github.com/0xShug0/audio.cpp / unicode_byte_to_utf8_map

Function unicode_byte_to_utf8_map

external/llama_tokenizer/unicode.cpp:148–170  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

146}
147
148static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {
149 std::unordered_map<uint8_t, std::string> map;
150 for (int ch = 0x21; ch <= 0x7E; ++ch) { // u'!' to u'~'
151 assert(0 <= ch && ch < 256);
152 map[ch] = unicode_cpt_to_utf8(ch);
153 }
154 for (int ch = 0xA1; ch <= 0xAC; ++ch) { // u'¡' to u'¬'
155 assert(0 <= ch && ch < 256);
156 map[ch] = unicode_cpt_to_utf8(ch);
157 }
158 for (int ch = 0xAE; ch <= 0xFF; ++ch) { // u'®' to u'ÿ'
159 assert(0 <= ch && ch < 256);
160 map[ch] = unicode_cpt_to_utf8(ch);
161 }
162 auto n = 0;
163 for (int ch = 0; ch < 256; ++ch) {
164 if (map.find(ch) == map.end()) {
165 map[ch] = unicode_cpt_to_utf8(256 + n);
166 ++n;
167 }
168 }
169 return map;
170}
171
172static std::unordered_map<std::string, uint8_t> unicode_utf8_to_byte_map() {
173 std::unordered_map<std::string, uint8_t> map;

Callers 1

unicode_byte_to_utf8Function · 0.85

Calls 3

unicode_cpt_to_utf8Function · 0.85
findMethod · 0.45
endMethod · 0.45

Tested by

no test coverage detected