| 7563 | // #define PRETOKENIZERDEBUG |
| 7564 | |
| 7565 | static void tokenizer_st_partition(const llama_vocab & vocab, std::forward_list<fragment_buffer_variant> & buffer) |
| 7566 | { |
| 7567 | // for each special token |
| 7568 | for (const auto & st: vocab.special_tokens_cache) { |
| 7569 | const auto & special_token = st.first; |
| 7570 | const auto & special_id = st.second; |
| 7571 | |
| 7572 | // for each text fragment |
| 7573 | std::forward_list<fragment_buffer_variant>::iterator it = buffer.begin(); |
| 7574 | while (it != buffer.end()) { |
| 7575 | auto & fragment = (*it); |
| 7576 | |
| 7577 | // if a fragment is text ( not yet processed ) |
| 7578 | if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_RAW_TEXT) { |
| 7579 | auto * raw_text = &(fragment.raw_text); |
| 7580 | |
| 7581 | auto raw_text_base_offset = fragment.offset; |
| 7582 | auto raw_text_base_length = fragment.length; |
| 7583 | |
| 7584 | // loop over the text |
| 7585 | while (true) { |
| 7586 | // find the first occurence of a given special token in this fragment |
| 7587 | // passing offset argument only limit the "search area" but match coordinates |
| 7588 | // are still relative to the source full raw_text |
| 7589 | auto match = raw_text->find(special_token, raw_text_base_offset); |
| 7590 | |
| 7591 | // no occurences found, stop processing this fragment for a given special token |
| 7592 | if (match == std::string::npos) break; |
| 7593 | |
| 7594 | // check if match is within bounds of offset <-> length |
| 7595 | if (match + special_token.length() > raw_text_base_offset + raw_text_base_length) break; |
| 7596 | |
| 7597 | #ifdef PRETOKENIZERDEBUG |
| 7598 | fprintf(stderr, "FF: (%ld %ld %ld) '%s'\n", raw_text->length(), raw_text_base_offset, raw_text_base_length, raw_text->substr(raw_text_base_offset, raw_text_base_length).c_str()); |
| 7599 | #endif |
| 7600 | auto source = std::distance(buffer.begin(), it); |
| 7601 | |
| 7602 | // if match is further than base offset |
| 7603 | // then we have some text to the left of it |
| 7604 | if (match > raw_text_base_offset) { |
| 7605 | // left |
| 7606 | const int64_t left_reminder_offset = raw_text_base_offset + 0; |
| 7607 | const int64_t left_reminder_length = match - raw_text_base_offset; |
| 7608 | buffer.emplace_after(it, (*raw_text), left_reminder_offset, left_reminder_length); |
| 7609 | |
| 7610 | #ifdef PRETOKENIZERDEBUG |
| 7611 | fprintf(stderr, "FL: (%ld %ld) '%s'\n", left_reminder_offset, left_reminder_length, raw_text->substr(left_reminder_offset, left_reminder_length).c_str()); |
| 7612 | #endif |
| 7613 | it++; |
| 7614 | } |
| 7615 | |
| 7616 | // special token |
| 7617 | buffer.emplace_after(it, special_id); |
| 7618 | it++; |
| 7619 | |
| 7620 | // right |
| 7621 | if (match + special_token.length() < raw_text_base_offset + raw_text_base_length) { |
| 7622 | const int64_t right_reminder_offset = match + special_token.length(); |