GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+
| 235 | |
| 236 | // GPT2 system regex: 's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+ |
| 237 | static std::vector<size_t> unicode_regex_split_custom_gpt2(const std::string & text, const std::vector<size_t> & offsets) { |
| 238 | std::vector<size_t> bpe_offsets; // store the offset of each word |
| 239 | bpe_offsets.reserve(offsets.size()); // Reserve memory for the approximate size |
| 240 | |
| 241 | const auto cpts = unicode_cpts_from_utf8(text); |
| 242 | |
| 243 | size_t start = 0; |
| 244 | for (auto offset : offsets) { |
| 245 | const size_t offset_ini = start; |
| 246 | const size_t offset_end = start + offset; |
| 247 | assert(offset_end <= cpts.size()); |
| 248 | start = offset_end; |
| 249 | |
| 250 | static const uint32_t OUT_OF_RANGE = 0xFFFFFFFF; |
| 251 | auto _get_cpt = [&] (const size_t pos) -> uint32_t { |
| 252 | return (offset_ini <= pos && pos < offset_end) ? cpts[pos] : OUT_OF_RANGE; |
| 253 | }; |
| 254 | |
| 255 | auto _get_flags = [&] (const size_t pos) -> unicode_cpt_flags { |
| 256 | return (offset_ini <= pos && pos < offset_end) ? unicode_cpt_flags_from_cpt(cpts[pos]) : unicode_cpt_flags{}; |
| 257 | }; |
| 258 | |
| 259 | size_t _prev_end = offset_ini; |
| 260 | auto _add_token = [&] (const size_t end) -> size_t { |
| 261 | assert(_prev_end <= end && end <= offset_end); |
| 262 | size_t len = end - _prev_end; |
| 263 | if (len > 0) { |
| 264 | bpe_offsets.push_back(len); |
| 265 | } |
| 266 | _prev_end = end; |
| 267 | //if (len > 0) { |
| 268 | // std::string s = ""; |
| 269 | // for(size_t p = end-len; p < end; p++) |
| 270 | // s += unicode_cpt_to_utf8(cpts[p]); |
| 271 | // printf(">>> '%s'\n", s.c_str()); |
| 272 | //} |
| 273 | return len; |
| 274 | }; |
| 275 | |
| 276 | for (size_t pos = offset_ini; pos < offset_end; /*pos++*/ ) { |
| 277 | const uint32_t cpt = _get_cpt(pos); |
| 278 | const auto flags = _get_flags(pos); |
| 279 | |
| 280 | // regex: 's|'t|'re|'ve|'m|'ll|'d |
| 281 | if (cpt == '\'' && pos+1 < offset_end) { |
| 282 | uint32_t cpt_next = _get_cpt(pos+1); |
| 283 | if (cpt_next == 's' || cpt_next == 't' || cpt_next == 'm' || cpt_next == 'd') { |
| 284 | pos += _add_token(pos+2); |
| 285 | continue; |
| 286 | } |
| 287 | if (pos+2 < offset_end) { |
| 288 | uint32_t cpt_next_next = _get_cpt(pos+2); |
| 289 | if ((cpt_next == 'r' && cpt_next_next == 'e') || |
| 290 | (cpt_next == 'v' && cpt_next_next == 'e') || |
| 291 | (cpt_next == 'l' && cpt_next_next == 'l')) { |
| 292 | pos += _add_token(pos+3); |
| 293 | continue; |
| 294 | } |
no test coverage detected