MCPcopy Create free account
hub / github.com/Tiiny-AI/PowerInfer / unicode_cpt_to_utf8

Function unicode_cpt_to_utf8

smallthinker/src/unicode.cpp:574–601  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

572//
573
574std::string unicode_cpt_to_utf8(uint32_t cpt) {
575 std::string result;
576
577 if (/* 0x00 <= cpt && */ cpt <= 0x7f) {
578 result.push_back(cpt);
579 return result;
580 }
581 if (0x80 <= cpt && cpt <= 0x7ff) {
582 result.push_back(0xc0 | ((cpt >> 6) & 0x1f));
583 result.push_back(0x80 | (cpt & 0x3f));
584 return result;
585 }
586 if (0x800 <= cpt && cpt <= 0xffff) {
587 result.push_back(0xe0 | ((cpt >> 12) & 0x0f));
588 result.push_back(0x80 | ((cpt >> 6) & 0x3f));
589 result.push_back(0x80 | (cpt & 0x3f));
590 return result;
591 }
592 if (0x10000 <= cpt && cpt <= 0x10ffff) {
593 result.push_back(0xf0 | ((cpt >> 18) & 0x07));
594 result.push_back(0x80 | ((cpt >> 12) & 0x3f));
595 result.push_back(0x80 | ((cpt >> 6) & 0x3f));
596 result.push_back(0x80 | (cpt & 0x3f));
597 return result;
598 }
599
600 throw std::invalid_argument("invalid codepoint");
601}
602
603std::vector<uint32_t> unicode_cpts_normalize_nfd(const std::vector<uint32_t> & cpts) {
604 auto comp = [] (const uint32_t cpt, const range_nfd & range) {

Callers 10

preprocessMethod · 0.85
llama_decode_textFunction · 0.85
unicode_cpts_to_utf8Function · 0.85
unicode_byte_to_utf8_mapFunction · 0.85
unicode_utf8_to_byte_mapFunction · 0.85
unicode_regex_splitFunction · 0.85
llama_grammar_validateFunction · 0.85
mainFunction · 0.85
mainFunction · 0.85

Calls 1

push_backMethod · 0.45

Tested by 3

llama_grammar_validateFunction · 0.68
mainFunction · 0.68
mainFunction · 0.68