Function MakeBaseModelProto

external/sentencepiece/src/model_interface_test.cc:31–48

    TrainerSpec::CHAR};

ModelProto MakeBaseModelProto(TrainerSpec::ModelType type,
                              bool byte_fallback = false) {
  ModelProto model_proto;
  auto *sp1 = model_proto.add_pieces();
  auto *sp2 = model_proto.add_pieces();
  auto *sp3 = model_proto.add_pieces();
  model_proto.mutable_trainer_spec()->set_model_type(type);
  model_proto.mutable_trainer_spec()->set_byte_fallback(byte_fallback);

  sp1->set_type(ModelProto::SentencePiece::UNKNOWN);
  sp1->set_piece("<unk>");
  sp2->set_type(ModelProto::SentencePiece::CONTROL);
  sp2->set_piece("<s>");
  sp3->set_type(ModelProto::SentencePiece::CONTROL);
  sp3->set_piece("</s>");

  return model_proto;
}

void AddPiece(ModelProto *model_proto, const std::string &piece,
              float score = 0.0) {