Function TokenizeString

tensorflow/lite/testing/tokenize_test.cc:36–41

};

std::vector<std::string> TokenizeString(const std::string& s) {
  std::stringstream ss(s);
  TokenCollector collector;
  Tokenize(&ss, &collector);
  return collector.Tokens();
}

TEST(TokenizeTest, TokenDetection) {
  EXPECT_THAT(TokenizeString("x :1"), ElementsAre("x", ":", "1"));