| 60 | } |
| 61 | |
| 62 | std::vector<SentencePiecePiece> extract_pieces(const sentencepiece::SentencePieceProcessor & processor) { |
| 63 | std::vector<SentencePiecePiece> pieces; |
| 64 | const int piece_count = processor.GetPieceSize(); |
| 65 | pieces.reserve(static_cast<size_t>(piece_count)); |
| 66 | for (int id = 0; id < piece_count; ++id) { |
| 67 | SentencePiecePiece piece; |
| 68 | piece.id = id; |
| 69 | piece.score = processor.GetScore(id); |
| 70 | piece.text = processor.IdToPiece(id); |
| 71 | if (processor.IsUnknown(id)) { |
| 72 | piece.type = SentencePieceType::Unknown; |
| 73 | } else if (processor.IsControl(id)) { |
| 74 | piece.type = SentencePieceType::Control; |
| 75 | } else if (processor.IsUnused(id)) { |
| 76 | piece.type = SentencePieceType::Unused; |
| 77 | } else if (processor.IsByte(id)) { |
| 78 | piece.type = SentencePieceType::Byte; |
| 79 | } else { |
| 80 | piece.type = SentencePieceType::Normal; |
| 81 | } |
| 82 | pieces.push_back(std::move(piece)); |
| 83 | } |
| 84 | return pieces; |
| 85 | } |
| 86 | |
| 87 | ProcessorPtr lookup_processor_for_pieces(const std::vector<SentencePiecePiece> & pieces) { |
| 88 | const auto fingerprint = fingerprint_pieces(pieces); |
no test coverage detected