Function load_processor

src/framework/tokenizers/sentencepiece.cpp:56–60

}

ProcessorPtr load_processor(const std::filesystem::path & model_path) {
    auto processor = std::make_shared<sentencepiece::SentencePieceProcessor>();
    require_status(processor->Load(model_path.string()), "failed to load SentencePiece model");
    return processor;
}

std::vector<SentencePiecePiece> extract_pieces(const sentencepiece::SentencePieceProcessor & processor) {
    std::vector<SentencePiecePiece> pieces;