Function getToken

src/expression/parse.js:291–487

* Get next token in the current string expr. * The token and token type are available as token and tokenType * @private

(state)

   * @private
   */
  function getToken (state) {
    state.tokenType = TOKENTYPE.NULL
    state.token = ''
    state.comment = ''

    // skip over ignored characters:
    while (true) {
      // comments:
      if (currentCharacter(state) === '#') {
        while (currentCharacter(state) !== '\n' &&
               currentCharacter(state) !== '') {
          state.comment += currentCharacter(state)
          next(state)
        }
      }
      // whitespace: space, tab, and newline when inside parameters
      if (parse.isWhitespace(currentCharacter(state), state.nestingLevel)) {
        next(state)
      } else {
        break
      }
    }

    // check for end of expression
    if (currentCharacter(state) === '') {
      // token is still empty
      state.tokenType = TOKENTYPE.DELIMITER
      return
    }

    // check for new line character
    if (currentCharacter(state) === '\n' && !state.nestingLevel) {
      state.tokenType = TOKENTYPE.DELIMITER
      state.token = currentCharacter(state)
      next(state)
      return
    }

    const c1 = currentCharacter(state)
    const c2 = currentString(state, 2)
    const c3 = currentString(state, 3)
    if (c3.length === 3 && DELIMITERS[c3]) {
      state.tokenType = TOKENTYPE.DELIMITER
      state.token = c3
      next(state)
      next(state)
      next(state)
      return
    }

    // check for delimiters consisting of 2 characters
    // Special case: the check for '?.' is to prevent a case like 'a?.3:.7' from being interpreted as optional chaining
    // TODO: refactor the tokenization into some better way to deal with cases like 'a?.3:.7', see https://github.com/josdejong/mathjs/pull/3584
    if (
      c2.length === 2 &&
      DELIMITERS[c2] &&
      (c2 !== '?.' || !parse.isDigit(state.expression.charAt(state.index + 2)))
    ) {