MCPcopy Create free account
hub / github.com/Tiiny-AI/PowerInfer / codepoint_to_utf8

Function codepoint_to_utf8

unicode.h:225–249  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

223};
224
225static std::string codepoint_to_utf8(uint32_t cp) {
226 std::string result;
227 if (/* 0x00 <= cp && */ cp <= 0x7f) {
228 result.push_back(cp);
229 }
230 else if (0x80 <= cp && cp <= 0x7ff) {
231 result.push_back(0xc0 | ((cp >> 6) & 0x1f));
232 result.push_back(0x80 | (cp & 0x3f));
233 }
234 else if (0x800 <= cp && cp <= 0xffff) {
235 result.push_back(0xe0 | ((cp >> 12) & 0x0f));
236 result.push_back(0x80 | ((cp >> 6) & 0x3f));
237 result.push_back(0x80 | (cp & 0x3f));
238 }
239 else if (0x10000 <= cp && cp <= 0x10ffff) {
240 result.push_back(0xf0 | ((cp >> 18) & 0x07));
241 result.push_back(0x80 | ((cp >> 12) & 0x3f));
242 result.push_back(0x80 | ((cp >> 6) & 0x3f));
243 result.push_back(0x80 | (cp & 0x3f));
244 }
245 else {
246 throw std::invalid_argument("invalid codepoint");
247 }
248 return result;
249}
250
251static std::string codepoints_to_utf8(const std::vector<uint32_t> & cps) {
252 std::string result;

Callers 7

bpe_gpt2_preprocessMethod · 0.85
llama_decode_textFunction · 0.85
codepoints_to_utf8Function · 0.85
bytes_to_unicode_map_bpeFunction · 0.85
unicode_to_bytes_map_bpeFunction · 0.85
mainFunction · 0.85
mainFunction · 0.85

Calls 1

push_backMethod · 0.45

Tested by 2

mainFunction · 0.68
mainFunction · 0.68