Method next

src/compute-engine/latex-syntax/tokenizer.ts:191–295

* Return the next token, or null.

()

      re.lastIndex = this.pos;
    } else {
      re.lastIndex =
        this.offsets![this.pos < this.s.length ? this.pos : this.s.length];
    }
    const execResult = re.exec(this.joined);
    if (execResult?.[0]) {
      this.pos += execResult[0].length;
      return execResult[0];
    }
    return null;
  }
  /**
   * Return the next token, or null.
   */
  next(): Token | null {
    // If we've reached the end, exit
    if (this.end()) return null;
    // Handle white space
    // In text mode, spaces are significant,
    // however they are coalesced unless \obeyspaces
    if (!this.obeyspaces && this.match(/^[ \f\n\r\t\v\xA0\u2028\u2029]+/)) {
      // Note that browsers are inconsistent in their definitions of the
      // `\s` metacharacter, so we use an explicit pattern instead.

      // - IE:          `[ \f\n\r\t\v]`
      // - Chrome:      `[ \f\n\r\t\v\u00A0]`
      // - Firefox:     `[ \f\n\r\t\v\u00A0\u2028\u2029]`
      // - \f \u000C: form feed (FORM FEED)
      // - \n \u000A: linefeed (LINE FEED)
      // - \r \u000D: carriage return
      // - \t \u0009: tab (CHARACTER TABULATION)
      // - \v \u000B: vertical tab (LINE TABULATION)
      // - \u00A0: NON-BREAKING SPACE
      // - \u2028: LINE SEPARATOR
      // - \u2029: PARAGRAPH SEPARATOR
      return '<space>';
    } else if (
      this.obeyspaces &&
      this.match(/^[ \f\n\r\t\v\xA0\u2028\u2029]/)
    ) {
      // Don't coalesce when this.obeyspaces is true (different regex from above)
      return '<space>';
    }
    const next = this.get();
    // Is it a command?
    if (next === '\\') {
      if (!this.end()) {
        // A command is either a string of letters (control word)...
        let command = this.match(/^[a-zA-Z]+/);
        if (command) {
          // Spaces after a 'control word' are ignored
          // (but not after a 'control symbol' (single char)
          this.match(/^[ \f\n\r\t\v\xA0\u2028\u2029]*/);
        } else {
          // ... or a single non-letter character (control char)
          command = this.get();
          if (command === ' ') {
            // The `\ ` command is equivalent to a single space
            return '<space>';