Function tokensToString

src/compute-engine/latex-syntax/tokenizer.ts:454–483

(
  tokens: Token | Token[] | [Token[] | Token][]
)

          discardedLength: line.length - m[0].length,
        });
      }
    }
    lineOffset += line.length + separator.length;
  }

  const tokenizer = new Tokenizer(stream);
  const result: Token[] = [];

  do result.push(...expand(tokenizer, args));
  while (!tokenizer.end());

  return result;
}

export function countTokens(s: string): number {
  return tokenize(s).length;
}

export function joinLatex(segments: Iterable<string>): string {
  let sep = '';
  let result = '';
  for (const segment of segments) {
    if (segment === undefined || segment === null) continue;
    if (typeof segment === 'string') {
      // If the segment begins with a char that *could* be in a command
      // name... insert a separator (if one was needed for the previous segment)
      if (/[a-zA-Z]/.test(segment[0])) result += sep;

      // If the segment ends in a command add a space before the next segment
      if (/\\[a-zA-Z]+\*?$/.test(segment)) sep = ' ';
      else sep = '';
    }
    result += segment.toString();