| 434 | } |
| 435 | |
| 436 | std::vector<uint32_t> SPTokenizer::encode(const std::string& text) const { |
| 437 | if (text.empty()) return {}; |
| 438 | |
| 439 | auto text_segments = split_with_special_tokens(text); |
| 440 | std::vector<uint32_t> token_ids; |
| 441 | |
| 442 | for (const auto& segment : text_segments) { |
| 443 | auto special_it = special_tokens_.find(segment); |
| 444 | if (special_it != special_tokens_.end()) { |
| 445 | token_ids.push_back(special_it->second); |
| 446 | } else { |
| 447 | std::string processed = preprocess_text(segment); |
| 448 | if (processed.empty()) continue; |
| 449 | |
| 450 | if (sp_bpe_mode_) { |
| 451 | auto ids = tokenize_with_bpe(processed); |
| 452 | token_ids.insert(token_ids.end(), ids.begin(), ids.end()); |
| 453 | } else { |
| 454 | auto token_pairs = tokenize_with_trie(processed); |
| 455 | for (const auto& [token, id] : token_pairs) { |
| 456 | token_ids.push_back(id); |
| 457 | } |
| 458 | } |
| 459 | } |
| 460 | } |
| 461 | |
| 462 | return token_ids; |
| 463 | } |
| 464 | |
| 465 | std::string SPTokenizer::decode(const std::vector<uint32_t>& tokens) const { |
| 466 | if (tokens.size() == 1) { |