Function fingerprint_piece

src/framework/tokenizers/sentencepiece.cpp:40–46

}

size_t fingerprint_piece(const SentencePiecePiece & piece) {
    size_t seed = std::hash<int>{}(piece.id);
    seed = hash_combine(seed, std::hash<std::string>{}(piece.text));
    seed = hash_combine(seed, std::hash<int>{}(static_cast<int>(piece.type)));
    seed = hash_combine(seed, std::hash<int32_t>{}(static_cast<int32_t>(piece.score * 1000000.0F)));
    return seed;
}

size_t fingerprint_pieces(const std::vector<SentencePiecePiece> & pieces) {
    size_t seed = std::hash<size_t>{}(pieces.size());