Encode text into tokens
| 167 | |
| 168 | // Encode text into tokens |
| 169 | std::vector<int> encode(const std::string& text) const |
| 170 | { |
| 171 | if (text.empty()) return {}; |
| 172 | |
| 173 | // Convert to initial tokens |
| 174 | std::vector<int> tokens; |
| 175 | tokens.reserve(text.size()); |
| 176 | for (unsigned char byte : text) { |
| 177 | tokens.push_back(static_cast<int>(byte)); |
| 178 | } |
| 179 | |
| 180 | // Apply all merges in order |
| 181 | for (size_t merge_idx = BPE_BASE_VOCAB_SIZE; merge_idx < merges.size(); merge_idx++) { |
| 182 | const Merge& m = merges[merge_idx]; |
| 183 | |
| 184 | std::vector<int> new_tokens; |
| 185 | new_tokens.reserve(tokens.size()); |
| 186 | |
| 187 | for (size_t i = 0; i < tokens.size(); ) { |
| 188 | if (i < tokens.size() - 1 && tokens[i] == m.left && tokens[i + 1] == m.right) { |
| 189 | new_tokens.push_back(m.token_id); |
| 190 | i += 2; |
| 191 | } |
| 192 | else { |
| 193 | new_tokens.push_back(tokens[i]); |
| 194 | i++; |
| 195 | } |
| 196 | } |
| 197 | |
| 198 | tokens = std::move(new_tokens); |
| 199 | } |
| 200 | |
| 201 | return tokens; |
| 202 | } |
| 203 | |
| 204 | // Decode tokens back to text |
| 205 | std::string decode(const std::vector<int>& tokens, bool display_special_tokens = true) const |