Function flattenTokens

packs/step-parser/src/tokenizer.ts:96–107

(tokens: NestedToken[])

// and flatten it so content is always a string
// and type the type of the leaf
function flattenTokens(tokens: NestedToken[]) {
  const flatList: FlatToken[] = [];
  tokens.forEach(token => {
    const { type, content } = token;
    if (Array.isArray(content)) {
      flatList.push(...flattenTokens(content));
    } else {
      flatList.push({ type, content });
    }
  });
  return flatList;
}

export class MissingGrammarError extends Error {
  lang: string;