| 561 | } |
| 562 | |
| 563 | std::vector<int32_t> tokenize_bpe(const BpeVocabulary & vocab, const std::string & text, bool parse_special) { |
| 564 | if (text.empty()) { |
| 565 | return {}; |
| 566 | } |
| 567 | llm_tokenizer_bpe tokenizer(vocab.pre_type); |
| 568 | llm_tokenizer_bpe_session session(vocab, tokenizer); |
| 569 | |
| 570 | std::forward_list<fragment_buffer_variant> fragment_buffer; |
| 571 | fragment_buffer.emplace_front(text, 0, static_cast<int64_t>(text.size())); |
| 572 | tokenizer_st_partition(vocab, fragment_buffer, parse_special); |
| 573 | |
| 574 | std::vector<int32_t> output; |
| 575 | for (const auto & fragment : fragment_buffer) { |
| 576 | if (fragment.type == FRAGMENT_BUFFER_VARIANT_TYPE_TOKEN) { |
| 577 | output.push_back(fragment.token); |
| 578 | } else { |
| 579 | session.tokenize( |
| 580 | fragment.raw_text.substr(static_cast<size_t>(fragment.offset), static_cast<size_t>(fragment.length)), |
| 581 | output); |
| 582 | } |
| 583 | } |
| 584 | return output; |
| 585 | } |
| 586 | |
| 587 | std::string decode_bpe(const BpeVocabulary & vocab, const std::vector<int32_t> & token_ids, bool skip_special_tokens) { |
| 588 | std::string decoded; |
no test coverage detected