Function Eval

tensorflow/lite/models/smartreply/ops/normalize.cc:58–97

static const int32_t kMaxInputChars = 300;

TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
  tflite::StringRef input = tflite::GetString(GetInput(context, node, 0), 0);

  string result(absl::AsciiStrToLower(absl::string_view(input.str, input.len)));
  absl::StripAsciiWhitespace(&result);
  // Do not remove commas, semi-colons or colons from the sentences as they can
  // indicate the beginning of a new clause.
  RE2::GlobalReplace(&result, kPunctuationsRegex, "");
  RE2::GlobalReplace(&result, "\\s('t|'nt|n't|'d|'ll|'s|'m|'ve|'re)([\\s,;:/])",
                     "\\1\\2");
  RE2::GlobalReplace(&result, "\\s('t|'nt|n't|'d|'ll|'s|'m|'ve|'re)$", "\\1");
  for (auto iter = kRegexTransforms->begin(); iter != kRegexTransforms->end();
       iter++) {
    RE2::GlobalReplace(&result, iter->first, iter->second);
  }

  // Treat questions & interjections as special cases.
  RE2::GlobalReplace(&result, "([?])+", "\\1");
  RE2::GlobalReplace(&result, "([!])+", "\\1");
  RE2::GlobalReplace(&result, "([^?!]+)([?!])", "\\1 \\2 ");
  RE2::GlobalReplace(&result, "([?!])([?!])", "\\1 \\2");

  RE2::GlobalReplace(&result, "[\\s,:;\\-&'\"]+$", "");
  RE2::GlobalReplace(&result, "^[\\s,:;\\-&'\"]+", "");
  absl::StripAsciiWhitespace(&result);

  // Add start and end token.
  // Truncate input to maximum allowed size.
  if (result.length() <= kMaxInputChars) {
    absl::StrAppend(&result, " ", kEndToken);
  } else {
    result = result.substr(0, kMaxInputChars);
  }
  result = absl::StrCat(kStartToken, " ", result);

  tflite::DynamicBuffer buf;
  buf.AddString(result.data(), result.length());
  buf.WriteToTensorAsVector(GetOutput(context, node, 0));
  return kTfLiteOk;
}

}  // namespace normalize