Method readToken

src/parser/token-reader.ts:135–185

()

  }

  private readToken(): Token {
    this.skipWhitespaceAndComments();

    const position = this.scanner.position;
    const byte = this.scanner.peek();

    if (byte === -1) {
      return { type: "eof", position };
    }

    // Name: /...
    if (byte === CHAR_SLASH) {
      return this.readName(position);
    }

    // Literal string: (...)
    if (byte === CHAR_PARENTHESIS_OPEN) {
      return this.readLiteralString(position);
    }

    // Hex string or dict delimiter: < or <<
    if (byte === CHAR_ANGLE_BRACKET_OPEN) {
      return this.readAngleBracket(position);
    }

    // Dict end or unexpected >
    if (byte === CHAR_ANGLE_BRACKET_CLOSE) {
      return this.readClosingAngle(position);
    }

    // Array delimiters
    if (byte === CHAR_SQUARE_BRACKET_OPEN) {
      this.scanner.advance();

      return { type: "delimiter", value: "[", position };
    }

    if (byte === CHAR_SQUARE_BRACKET_CLOSE) {
      this.scanner.advance();

      return { type: "delimiter", value: "]", position };
    }

    // Number: digit, +, -, or .
    if (this.isNumberStart(byte)) {
      return this.readNumber(position);
    }

    // Keyword or unknown
    return this.readKeyword(position);
  }

  private isNumberStart(byte: number): boolean {
    return (