Method Encode

external/sentencepiece/src/word_model.cc:28–39

Model::~Model() {}

EncodeResult Model::Encode(absl::string_view normalized) const {
  if (!status().ok() || normalized.empty()) {
    return {};
  }

  EncodeResult output;
  for (const auto &w : SplitIntoWords(normalized)) {
    output.emplace_back(w, PieceToId(w));
  }

  return output;
}

}  // namespace word
}  // namespace sentencepiece