Function fingerprint_pieces

src/framework/tokenizers/sentencepiece.cpp:48–54

}

size_t fingerprint_pieces(const std::vector<SentencePiecePiece> & pieces) {
    size_t seed = std::hash<size_t>{}(pieces.size());
    for (const auto & piece : pieces) {
        seed = hash_combine(seed, fingerprint_piece(piece));
    }
    return seed;
}

ProcessorPtr load_processor(const std::filesystem::path & model_path) {
    auto processor = std::make_shared<sentencepiece::SentencePieceProcessor>();