LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
| 353 | |
| 354 | // LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" |
| 355 | static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) { |
| 356 | std::vector<size_t> bpe_offsets; // store the offset of each word |
| 357 | bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size |
| 358 | |
| 359 | const auto cpts = unicode_cpts_from_utf8(text); |
| 360 | |
| 361 | size_t start = 0; |
| 362 | for (auto offset : offsets) { |
| 363 | const size_t offset_ini = start; |
| 364 | const size_t offset_end = start + offset; |
| 365 | assert(offset_end <= cpts.size()); |
| 366 | start = offset_end; |
| 367 | |
| 368 | static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; |
| 369 | auto _get_cpt = [&] (const size_t pos) -> uint32_t { |
| 370 | return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; |
| 371 | }; |
| 372 | |
| 373 | auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { |
| 374 | return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; |
| 375 | }; |
| 376 | |
| 377 | size_t _prev_end = offset_ini; |
| 378 | auto _add_token = [&] (const size_t end) -> size_t { |
| 379 | assert(_prev_end <= end && end <= offset_end); |
| 380 | size_t len = end - _prev_end; |
| 381 | if (len > 0) { |
| 382 | bpe_offsets.push_back(len); |
| 383 | } |
| 384 | _prev_end = end; |
| 385 | //if (len > 0) { |
| 386 | // std::string s = ""; |
| 387 | // for(size_t p = end-len; p < end; p++) |
| 388 | // s += unicode_cpt_to_utf8(cpts[p]); |
| 389 | // printf(">>> '%s'\n", s.c_str()); |
| 390 | //} |
| 391 | return len; |
| 392 | }; |
| 393 | |
| 394 | for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { |
| 395 | const uint32_t cpt = _get_cpt(pos); |
| 396 | const auto flags = _get_flags(pos); |
| 397 | |
| 398 | // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive |
| 399 | if (cpt == '\'' && pos+1 < offset_end) { |
| 400 | uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); |
| 401 | if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { |
| 402 | pos += _add_token(pos+2); |
| 403 | continue; |
| 404 | } |
| 405 | if (pos+2 < offset_end) { |
| 406 | uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); |
| 407 | if ((cpt_next == 'r' && cpt_next_next == 'e') || |
| 408 | (cpt_next == 'v' && cpt_next_next == 'e') || |
| 409 | (cpt_next == 'l' && cpt_next_next == 'l')) { |
| 410 | pos += _add_token(pos+3); |
| 411 | continue; |
| 412 | } |
no test coverage detected