Method tokenize

Rewrite-Parser.beta.js:2923–2969

()

    }

    tokenize() {
      // console.log('=== 开始词法分析 ===')
      while (this.position < this.input.length) {
        let currentChar = this.input[this.position]

        if (this.isWhitespace(currentChar)) {
          this.position++
          continue
        }

        if (currentChar === '(') {
          this.tokens.push({ type: 'LPAREN', value: '(' })
          // console.log(`Token: LPAREN '(' at position ${this.position}`)
          this.position++
          continue
        }

        if (currentChar === ')') {
          this.tokens.push({ type: 'RPAREN', value: ')' })
          // console.log(`Token: RPAREN ')' at position ${this.position}`)
          this.position++
          continue
        }

        if (currentChar === ',') {
          this.tokens.push({ type: 'COMMA', value: ',' })
          // console.log(`Token: COMMA ',' at position ${this.position}`)
          this.position++
          continue
        }

        // 收集单词
        let start = this.position
        while (
          this.position < this.input.length &&
          !this.isWhitespace(this.input[this.position]) &&
          !this.isDelimiter(this.input[this.position])
        ) {
          this.position++
        }
        let value = this.input.slice(start, this.position)
        this.tokens.push({ type: 'WORD', value })
        // console.log(`Token: WORD '${value}' from position ${start} to ${this.position}`)
      }
      // console.log('=== 词法分析完成 ===')
      return this.tokens
    }
  }

  // 语法分析器