Function tokenize

packages/compiler/src/ml_parser/lexer.ts:109–122

(
  source: string,
  url: string,
  getTagDefinition: (tagName: string) => TagDefinition,
  options: TokenizeOptions = {},
)

}

export function tokenize(
  source: string,
  url: string,
  getTagDefinition: (tagName: string) => TagDefinition,
  options: TokenizeOptions = {},
): TokenizeResult {
  const tokenizer = new _Tokenizer(new ParseSourceFile(source, url), getTagDefinition, options);
  tokenizer.tokenize();
  return new TokenizeResult(
    mergeTextTokens(tokenizer.tokens),
    tokenizer.errors,
    tokenizer.nonNormalizedIcuExpressions,
  );
}

const _CR_OR_CRLF_REGEXP = /\r\n?/g;