Function tokenize

src/core/tokenizer.ts:56–61

(
  text: string,
  mapfn: (v: RegExpMatchArray, k: number) => T
)

export const TOKEN_MATCHER = new RegExp(REGEX, "gu");

export function tokenize<T>(
  text: string,
  mapfn: (v: RegExpMatchArray, k: number) => T
) {
  return matchAll(text, TOKEN_MATCHER, mapfn);
}

//https://stackoverflow.com/a/6969486
function escapeRegExp(string: string) {