| 612 | } |
| 613 | |
| 614 | util::Status TrainerInterface::Serialize(ModelProto *model_proto) const { |
| 615 | RETURN_IF_ERROR(status()); |
| 616 | |
| 617 | // Duplicated sentencepiece is not allowed. |
| 618 | std::set<std::string> dup; |
| 619 | |
| 620 | model_proto->Clear(); |
| 621 | |
| 622 | #define CHECK_PIECE(piece) \ |
| 623 | CHECK_OR_RETURN(string_util::IsStructurallyValid(piece)); \ |
| 624 | CHECK_OR_RETURN(!piece.empty()); \ |
| 625 | CHECK_OR_RETURN(dup.insert(piece).second) << piece << " is already defined"; |
| 626 | |
| 627 | size_t fid = 0; |
| 628 | for (int id = 0; id < trainer_spec_.vocab_size(); ++id) { |
| 629 | const auto it = meta_pieces_.find(id); |
| 630 | if (it != meta_pieces_.end()) { |
| 631 | auto *sp = model_proto->add_pieces(); |
| 632 | sp->set_piece(it->second.first); |
| 633 | sp->set_type(it->second.second); |
| 634 | sp->set_score(0.0); |
| 635 | CHECK_EQ_OR_RETURN(model_proto->pieces_size() - 1, it->first); |
| 636 | CHECK_NE_OR_RETURN(ModelProto::SentencePiece::NORMAL, sp->type()); |
| 637 | CHECK_PIECE(sp->piece()); |
| 638 | } else if (fid < final_pieces_.size()) { |
| 639 | const auto &w = final_pieces_[fid++]; |
| 640 | auto *sp = model_proto->add_pieces(); |
| 641 | sp->set_piece(w.first); |
| 642 | sp->set_score(w.second); |
| 643 | CHECK_PIECE(sp->piece()); |
| 644 | } |
| 645 | } |
| 646 | |
| 647 | CHECK_EQ_OR_RETURN(fid, final_pieces_.size()); |
| 648 | |
| 649 | *(model_proto->mutable_trainer_spec()) = trainer_spec_; |
| 650 | *(model_proto->mutable_normalizer_spec()) = normalizer_spec_; |
| 651 | |
| 652 | if (!denormalizer_spec_.normalization_rule_tsv().empty()) { |
| 653 | *(model_proto->mutable_denormalizer_spec()) = denormalizer_spec_; |
| 654 | } |
| 655 | |
| 656 | if (!trainer_spec_.hard_vocab_limit() || |
| 657 | trainer_spec_.model_type() == TrainerSpec::CHAR) { |
| 658 | CHECK_GE_OR_RETURN(trainer_spec_.vocab_size(), model_proto->pieces_size()); |
| 659 | CHECK_GE_OR_RETURN(trainer_spec_.vocab_size(), |
| 660 | static_cast<int32_t>(dup.size())); |
| 661 | model_proto->mutable_trainer_spec()->set_vocab_size( |
| 662 | model_proto->pieces_size()); |
| 663 | } else { |
| 664 | CHECK_EQ_OR_RETURN(trainer_spec_.vocab_size(), model_proto->pieces_size()) |
| 665 | << absl::StrFormat( |
| 666 | "Vocabulary size too high (%d). Please set it to a value <= %d.", |
| 667 | trainer_spec_.vocab_size(), model_proto->pieces_size()); |
| 668 | CHECK_EQ_OR_RETURN(trainer_spec_.vocab_size(), |
| 669 | static_cast<int32_t>(dup.size())); |
| 670 | } |
| 671 | |