| 738 | } |
| 739 | |
| 740 | util::Status TrainerInterface::InitMetaPieces() { |
| 741 | CHECK_OR_RETURN(meta_pieces_.empty()); |
| 742 | bool has_unk = false; |
| 743 | |
| 744 | auto insert_id = [&has_unk, this](int id, const std::string &w) -> bool { |
| 745 | if (id < 0) return true; |
| 746 | if (id >= trainer_spec_.vocab_size() || |
| 747 | meta_pieces_.find(id) != meta_pieces_.end() || |
| 748 | (has_unk && w == trainer_spec_.unk_piece())) |
| 749 | return false; |
| 750 | if (w == trainer_spec_.unk_piece()) has_unk = true; |
| 751 | meta_pieces_[id] = std::make_pair( |
| 752 | w, w == trainer_spec_.unk_piece() ? ModelProto::SentencePiece::UNKNOWN |
| 753 | : ModelProto::SentencePiece::CONTROL); |
| 754 | return true; |
| 755 | }; |
| 756 | |
| 757 | CHECK_OR_RETURN(insert_id(trainer_spec_.unk_id(), trainer_spec_.unk_piece())); |
| 758 | CHECK_OR_RETURN(insert_id(trainer_spec_.bos_id(), trainer_spec_.bos_piece())); |
| 759 | CHECK_OR_RETURN(insert_id(trainer_spec_.eos_id(), trainer_spec_.eos_piece())); |
| 760 | CHECK_OR_RETURN(insert_id(trainer_spec_.pad_id(), trainer_spec_.pad_piece())); |
| 761 | |
| 762 | CHECK_OR_RETURN(has_unk) << trainer_spec_.unk_piece() << " must be defined."; |
| 763 | |
| 764 | std::set<std::string> dup; |
| 765 | |
| 766 | int id = 0; |
| 767 | auto insert_meta_symbol = |
| 768 | [&id, &dup, this](const std::string &w, |
| 769 | ModelProto::SentencePiece::Type type) -> util::Status { |
| 770 | if (!dup.insert(w).second) { |
| 771 | return util::InternalError(absl::StrCat( |
| 772 | w, " is already defined. duplicated symbols are not allowed.")); |
| 773 | } |
| 774 | |
| 775 | if (w == trainer_spec_.unk_piece()) { |
| 776 | return util::InternalError( |
| 777 | absl::StrCat(trainer_spec_.unk_piece(), |
| 778 | " must not be defined with --control_symbols and " |
| 779 | "--user_defined_symbols.")); |
| 780 | } |
| 781 | |
| 782 | if (w == trainer_spec_.bos_piece() && trainer_spec_.bos_id() >= 0) { |
| 783 | meta_pieces_[trainer_spec_.bos_id()].second = type; |
| 784 | } else if (w == trainer_spec_.eos_piece() && trainer_spec_.eos_id() >= 0) { |
| 785 | meta_pieces_[trainer_spec_.eos_id()].second = type; |
| 786 | } else if (w == trainer_spec_.pad_piece() && trainer_spec_.pad_id() >= 0) { |
| 787 | meta_pieces_[trainer_spec_.pad_id()].second = type; |
| 788 | } else { |
| 789 | while (meta_pieces_.find(id) != meta_pieces_.end()) ++id; |
| 790 | meta_pieces_[id] = std::make_pair(w, type); |
| 791 | } |
| 792 | |
| 793 | return util::OkStatus(); |
| 794 | }; |
| 795 | |
| 796 | for (const auto &w : trainer_spec_.control_symbols()) { |
| 797 | RETURN_IF_ERROR(insert_meta_symbol(w, ModelProto::SentencePiece::CONTROL)); |
nothing calls this directly
no test coverage detected