Method GetPrecompiledCharsMap

external/sentencepiece/src/builder.cc:287–323

static

// static
util::Status Builder::GetPrecompiledCharsMap(absl::string_view name,
                                             std::string *output) {
  CHECK_OR_RETURN(output);

  if (name == "identity") {
    output->clear();
    return util::OkStatus();
  }

  std::string result;

#ifndef DISABLE_EMBEDDED_DATA
  for (size_t i = 0; i < kNormalizationRules_size; ++i) {
    const auto *blob = &kNormalizationRules_blob[i];
    if (blob->name == name) {
      output->assign(blob->data, blob->size);
      CHECK_OR_RETURN(IsValidNormalizerData(*output));
      return util::OkStatus();
    }
  }
#else   // DISABLE_EMBEDDED_DATA
  {
    const std::string filename =
        absl::StrCat(util::JoinPath(GetDataDir(), name), ".bin");
    auto input = filesystem::NewReadableFile(filename, true /* is binary */);
    if (input->status().ok()) {
      input->ReadAll(output);
      CHECK_OR_RETURN(IsValidNormalizerData(*output));
      return util::OkStatus();
    }
  }
#endif  // DISABLE_EMBEDDED_DATA

  return util::StatusBuilder(util::StatusCode::kNotFound, GTL_LOC)
         << "No precompiled charsmap is found: " << name << " in "
         << GetDataDir();
}

#ifdef ENABLE_NFKC_COMPILE
namespace {