MCPcopy Create free account
hub / github.com/Tiiny-AI/PowerInfer / unicode_cpt_flags_array

Function unicode_cpt_flags_array

smallthinker/src/unicode.cpp:122–152  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

120//}
121
122static std::vector<unicode_cpt_flags> unicode_cpt_flags_array() {
123 std::vector<unicode_cpt_flags> cpt_flags(MAX_CODEPOINTS, unicode_cpt_flags::UNDEFINED);
124
125 assert (unicode_ranges_flags.begin()[0].first == 0);
126 assert (unicode_ranges_flags.begin()[unicode_ranges_flags.size()-1].first == MAX_CODEPOINTS);
127 for (size_t i = 1; i < unicode_ranges_flags.size(); ++i) {
128 const auto range_ini = unicode_ranges_flags.begin()[i-1]; // codepoint_ini, flags
129 const auto range_end = unicode_ranges_flags.begin()[i]; // codepoint_end, flags
130 for (uint32_t cpt = range_ini.first; cpt < range_end.first; ++cpt) {
131 cpt_flags[cpt] = range_ini.second;
132 }
133 }
134
135 for (auto cpt : unicode_set_whitespace) {
136 cpt_flags[cpt].is_whitespace = true;
137 }
138
139 for (auto p : unicode_map_lowercase) {
140 cpt_flags[p.second].is_lowercase = true;
141 }
142
143 for (auto p : unicode_map_uppercase) {
144 cpt_flags[p.second].is_uppercase = true;
145 }
146
147 for (auto &range : unicode_ranges_nfd) { // start, last, nfd
148 cpt_flags[range.nfd].is_nfd = true;
149 }
150
151 return cpt_flags;
152}
153
154static std::unordered_map<uint8_t, std::string> unicode_byte_to_utf8_map() {
155 std::unordered_map<uint8_t, std::string> map;

Callers 1

Calls 2

beginMethod · 0.45
sizeMethod · 0.45

Tested by

no test coverage detected