Function ToPieceArray

external/sentencepiece/src/sentencepiece_processor.cc:57–61

const char kReplacementCharacter[] = "\xef\xbf\xbd";

std::vector<absl::string_view> ToPieceArray(const std::vector<std::string> &v) {
  std::vector<absl::string_view> out(v.size());
  for (int i = 0; i < v.size(); ++i) out[i] = v[i];
  return out;
}

void ConvertToUnicodeSpansInternal(SentencePieceText *spt) {
  if (spt == nullptr || spt->text().empty()) return;