| 103 | } // namespace |
| 104 | |
| 105 | std::vector<SentencePiecePiece> load_sentencepiece_model(const std::filesystem::path & model_path) { |
| 106 | auto processor = load_processor(model_path); |
| 107 | auto pieces = extract_pieces(*processor); |
| 108 | cache_processor_for_pieces(pieces, std::move(processor)); |
| 109 | return pieces; |
| 110 | } |
| 111 | |
| 112 | std::vector<int32_t> tokenize_sentencepiece( |
| 113 | const std::vector<SentencePiecePiece> & pieces, |