Function tokenize

packs/step-parser/src/tokenizer.ts:7–52

(code: string, lang: string)

const newlineRe = /\r\n|\r|\n/;

export function tokenize(code: string, lang: string) {
  const grammar = Prism.languages[lang];
  if (!grammar) {
    throw new MissingGrammarError(lang);
  }

  const prismTokens = Prism.tokenize(code, Prism.languages[lang]);
  const nestedTokens = tokenizeStrings(prismTokens);
  const tokens = flattenTokens(nestedTokens);

  let currentLine: FlatToken[] = [];
  let currentTokenLine: string[] = [];
  let currentTypeLine: string[] = [];

  const lines = [currentLine];
  const tokenLines = [currentTokenLine];
  const typeLines = [currentTypeLine];

  tokens.forEach(token => {
    const contentLines = token.content.split(newlineRe);

    const firstContent = contentLines.shift();
    if (firstContent !== undefined && firstContent !== "") {
      currentLine.push({ type: token.type, content: firstContent });
      currentTokenLine.push(firstContent);
      currentTypeLine.push(token.type);
    }
    contentLines.forEach(content => {
      currentLine = [];
      currentTokenLine = [];
      currentTypeLine = [];
      lines.push(currentLine);
      tokenLines.push(currentTokenLine);
      typeLines.push(currentTypeLine);
      if (content !== "") {
        currentLine.push({ type: token.type, content });
        currentTokenLine.push(content);
        currentTypeLine.push(token.type);
      }
    });
  });
  return {
    tokens: tokenLines,
    types: typeLines
  };
}

type NestedToken = {
  type: string;