Function tokenize

src/compute-engine/latex-syntax/tokenizer.ts:402–423

(s: string, args: string[] = [])

      tokenize(argMap?.[param] ?? argMap?.['?'] ?? '\\placeholder{}', args)
    );
  } else {
    result.push(token);
  }

  return result;
}

/**
 * Create Tokens from a stream of LaTeX
 *
 * @param s - A string of LaTeX. It can include comments (with the `%`
 * marker) and multiple lines.
 */
export function tokenize(
  s: string,
  args: string[] = [],
  comments?: DiscardedComment[]
): Token[] {
  // Merge multiple lines into one, and remove comments.
  // Split keeping the line separators (capture group) so the exact separator
  // length — `\n` (1) or `\r\n` (2) — advances the original-input offset used
  // to report discarded-comment spans. `parts` alternates line, separator,
  // line, separator, …; even indices are lines, odd indices are the following
  // separator (or `undefined` for the last line).
  const str = s.toString();