Function tokenize

script/helpers/remove-liquid-statements.js:13–16

(str)

const removeConditionals = 'removeConditionals'

const tokenize = (str) => {
  const tokenizer = new Tokenizer(str)
  return tokenizer.readTopLevelTokens()
}
// This module is used by script/enterprise-server-deprecations/remove-version-markup.js to remove
// and update Liquid conditionals when a GHES release is being deprecated. It is also used by
// tests/content/remove-liquid-statements.js.