Function Tokenize

tensorflow/lite/testing/tokenize.cc:23–92

namespace testing {

void Tokenize(std::istream* input, TokenProcessor* processor) {
  enum State { kBuildQuotedToken, kBuildToken, kIdle };

  std::string current_token;
  State state = kIdle;
  auto start_token = [&](char c) {
    state = kBuildToken;
    current_token.clear();
    current_token = c;
  };
  auto issue_token = [&]() {
    state = kIdle;
    processor->ConsumeToken(&current_token);
    current_token.clear();
  };
  auto start_quoted_token = [&]() {
    state = kBuildQuotedToken;
    current_token.clear();
  };
  auto issue_quoted_token = [&]() {
    state = kIdle;
    processor->ConsumeToken(&current_token);
    current_token.clear();
  };
  auto issue_delim = [&](char d) {
    current_token = string(1, d);
    processor->ConsumeToken(&current_token);
    current_token.clear();
  };
  auto is_delim = [](char c) { return c == '{' || c == '}' || c == ':'; };
  auto is_quote = [](char c) { return c == '"'; };

  for (auto it = std::istreambuf_iterator<char>(*input);
       it != std::istreambuf_iterator<char>(); ++it) {
    switch (state) {
      case kIdle:
        if (is_delim(*it)) {
          issue_delim(*it);
        } else if (is_quote(*it)) {
          start_quoted_token();
        } else if (!isspace(*it)) {
          start_token(*it);
        }
        break;
      case kBuildToken:
        if (is_delim(*it)) {
          issue_token();
          issue_delim(*it);
        } else if (is_quote(*it)) {
          issue_token();
          start_quoted_token();
        } else if (isspace(*it)) {
          issue_token();
        } else {
          current_token += *it;
        }
        break;
      case kBuildQuotedToken: