LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"
| 331 | |
| 332 | // LLAMA3 system regex: "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+" |
| 333 | static std::vector<size_t> unicode_regex_split_custom_llama3(const std::string & text, const std::vector<size_t> & offsets) { |
| 334 | std::vector<size_t> bpe_offsets; // store the offset of each word |
| 335 | bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size |
| 336 | |
| 337 | const auto cpts = unicode_cpts_from_utf8(text); |
| 338 | |
| 339 | size_t start = 0; |
| 340 | for (auto offset : offsets) { |
| 341 | const size_t offset_ini = start; |
| 342 | const size_t offset_end = start + offset; |
| 343 | assert(offset_end <= cpts.size()); |
| 344 | start = offset_end; |
| 345 | |
| 346 | static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; |
| 347 | auto _get_cpt = [&] (const size_t pos) -> uint32_t { |
| 348 | return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; |
| 349 | }; |
| 350 | |
| 351 | auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { |
| 352 | return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; |
| 353 | }; |
| 354 | |
| 355 | size_t _prev_end = offset_ini; |
| 356 | auto _add_token = [&] (const size_t end) -> size_t { |
| 357 | assert(_prev_end <= end && end <= offset_end); |
| 358 | size_t len = end - _prev_end; |
| 359 | if (len > 0) { |
| 360 | bpe_offsets.push_back(len); |
| 361 | } |
| 362 | _prev_end = end; |
| 363 | //if (len > 0) { |
| 364 | // std::string s = ""; |
| 365 | // for(size_t p = end-len; p < end; p++) |
| 366 | // s += unicode_cpt_to_utf8(cpts[p]); |
| 367 | // printf(">>> '%s'\n", s.c_str()); |
| 368 | //} |
| 369 | return len; |
| 370 | }; |
| 371 | |
| 372 | for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { |
| 373 | const uint32_t cpt = _get_cpt(pos); |
| 374 | const auto flags = _get_flags(pos); |
| 375 | |
| 376 | // regex: (?i:'s|'t|'re|'ve|'m|'ll|'d) // case insensitive |
| 377 | if (cpt == '\'' && pos+1 < offset_end) { |
| 378 | uint32_t cpt_next = unicode_tolower(_get_cpt(pos+1)); |
| 379 | if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { |
| 380 | pos += _add_token(pos+2); |
| 381 | continue; |
| 382 | } |
| 383 | if (pos+2 < offset_end) { |
| 384 | uint32_t cpt_next_next = unicode_tolower(_get_cpt(pos+2)); |
| 385 | if ((cpt_next == 'r' && cpt_next_next == 'e') || |
| 386 | (cpt_next == 'v' && cpt_next_next == 'e') || |
| 387 | (cpt_next == 'l' && cpt_next_next == 'l')) { |
| 388 | pos += _add_token(pos+3); |
| 389 | continue; |
| 390 | } |
no test coverage detected