GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
| 213 | |
| 214 | // GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ |
| 215 | static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) { |
| 216 | std::vector<size_t> bpe_offsets; // store the offset of each word |
| 217 | bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size |
| 218 | |
| 219 | const auto cpts = unicode_cpts_from_utf8(text); |
| 220 | |
| 221 | size_t start = 0; |
| 222 | for (auto offset : offsets) { |
| 223 | const size_t offset_ini = start; |
| 224 | const size_t offset_end = start + offset; |
| 225 | assert(offset_end <= cpts.size()); |
| 226 | start = offset_end; |
| 227 | |
| 228 | static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; |
| 229 | auto _get_cpt = [&] (const size_t pos) -> uint32_t { |
| 230 | return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; |
| 231 | }; |
| 232 | |
| 233 | auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { |
| 234 | return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; |
| 235 | }; |
| 236 | |
| 237 | size_t _prev_end = offset_ini; |
| 238 | auto _add_token = [&] (const size_t end) -> size_t { |
| 239 | assert(_prev_end <= end && end <= offset_end); |
| 240 | size_t len = end - _prev_end; |
| 241 | if (len > 0) { |
| 242 | bpe_offsets.push_back(len); |
| 243 | } |
| 244 | _prev_end = end; |
| 245 | //if (len > 0) { |
| 246 | // std::string s = ""; |
| 247 | // for(size_t p = end-len; p < end; p++) |
| 248 | // s += unicode_cpt_to_utf8(cpts[p]); |
| 249 | // printf(">>> '%s'\n", s.c_str()); |
| 250 | //} |
| 251 | return len; |
| 252 | }; |
| 253 | |
| 254 | for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { |
| 255 | const uint32_t cpt = _get_cpt(pos); |
| 256 | const auto flags = _get_flags(pos); |
| 257 | |
| 258 | // regex: 's|'t|'re|'ve|'m|'ll|'d |
| 259 | if (cpt == '\'' && pos+1 < offset_end) { |
| 260 | uint32_t cpt_next = _get_cpt(pos+1); |
| 261 | if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { |
| 262 | pos += _add_token(pos+2); |
| 263 | continue; |
| 264 | } |
| 265 | if (pos+2 < offset_end) { |
| 266 | uint32_t cpt_next_next = _get_cpt(pos+2); |
| 267 | if ((cpt_next == 'r' && cpt_next_next == 'e') || |
| 268 | (cpt_next == 'v' && cpt_next_next == 'e') || |
| 269 | (cpt_next == 'l' && cpt_next_next == 'l')) { |
| 270 | pos += _add_token(pos+3); |
| 271 | continue; |
| 272 | } |
no test coverage detected