| 7105 | llm_tokenizer_spm(const llama_vocab & vocab): vocab(vocab) {} |
| 7106 | |
| 7107 | void tokenize(const std::string & text, std::vector<llama_vocab::id> & output) { |
| 7108 | // split string into utf8 chars |
| 7109 | int index = 0; |
| 7110 | size_t offs = 0; |
| 7111 | while (offs < text.size()) { |
| 7112 | llm_symbol sym; |
| 7113 | size_t len = utf8_len(text[offs]); |
| 7114 | sym.text = text.c_str() + offs; |
| 7115 | sym.n = std::min(len, text.size() - offs); |
| 7116 | offs += sym.n; |
| 7117 | sym.prev = index - 1; |
| 7118 | sym.next = offs == text.size() ? -1 : index + 1; |
| 7119 | index++; |
| 7120 | symbols.emplace_back(sym); |
| 7121 | } |
| 7122 | |
| 7123 | // seed the work queue with all possible 2-character tokens. |
| 7124 | for (size_t i = 1; i < symbols.size(); ++i) { |
| 7125 | try_add_bigram(i - 1, i); |
| 7126 | } |
| 7127 | |
| 7128 | // keep substituting the highest frequency pairs for as long as we can. |
| 7129 | while (!work_queue.empty()) { |
| 7130 | auto bigram = work_queue.top(); |
| 7131 | work_queue.pop(); |
| 7132 | |
| 7133 | auto & left_sym = symbols[bigram.left]; |
| 7134 | auto & right_sym = symbols[bigram.right]; |
| 7135 | |
| 7136 | // if one of the symbols already got merged, skip it. |
| 7137 | if (left_sym.n == 0 || right_sym.n == 0 || |
| 7138 | left_sym.n + right_sym.n != bigram.size) { |
| 7139 | continue; |
| 7140 | } |
| 7141 | |
| 7142 | // merge the right sym into the left one |
| 7143 | left_sym.n += right_sym.n; |
| 7144 | right_sym.n = 0; |
| 7145 | |
| 7146 | //LLAMA_LOG_INFO("left = '%*s' size = %zu\n", (int) left_sym.n, left_sym.text, bigram.size); |
| 7147 | |
| 7148 | // remove the right sym from the chain |
| 7149 | left_sym.next = right_sym.next; |
| 7150 | if (right_sym.next >= 0) { |
| 7151 | symbols[right_sym.next].prev = bigram.left; |
| 7152 | } |
| 7153 | |
| 7154 | // find more substitutions |
| 7155 | try_add_bigram(left_sym.prev, bigram.left); |
| 7156 | try_add_bigram(bigram.left, left_sym.next); |
| 7157 | } |
| 7158 | |
| 7159 | for (int i = 0; i != -1; i = symbols[i].next) { |
| 7160 | auto & symbol = symbols[i]; |
| 7161 | resegment(symbol, output); |
| 7162 | } |
| 7163 | } |
| 7164 | |