Class Tokenizer

Rewrite-Parser.beta.js:2908–2970

function parseRule(input) {
  // 分析器
  class Tokenizer {
    constructor(input) {
      this.input = input
      this.position = 0
      this.tokens = []
    }

    isWhitespace(char) {
      return /\s/.test(char)
    }

    isDelimiter(char) {
      return ['(', ')', ','].includes(char)
    }

    tokenize() {
      // console.log('=== 开始词法分析 ===')
      while (this.position < this.input.length) {
        let currentChar = this.input[this.position]

        if (this.isWhitespace(currentChar)) {
          this.position++
          continue
        }

        if (currentChar === '(') {
          this.tokens.push({ type: 'LPAREN', value: '(' })
          // console.log(`Token: LPAREN '(' at position ${this.position}`)
          this.position++
          continue
        }

        if (currentChar === ')') {
          this.tokens.push({ type: 'RPAREN', value: ')' })
          // console.log(`Token: RPAREN ')' at position ${this.position}`)
          this.position++
          continue
        }

        if (currentChar === ',') {
          this.tokens.push({ type: 'COMMA', value: ',' })
          // console.log(`Token: COMMA ',' at position ${this.position}`)
          this.position++
          continue
        }

        // 收集单词
        let start = this.position
        while (
          this.position < this.input.length &&
          !this.isWhitespace(this.input[this.position]) &&
          !this.isDelimiter(this.input[this.position])
        ) {
          this.position++
        }
        let value = this.input.slice(start, this.position)
        this.tokens.push({ type: 'WORD', value })
        // console.log(`Token: WORD '${value}' from position ${start} to ${this.position}`)