Method scan_token

aiscript-lexer/src/lib.rs:465–654

Scans the next token from the source

(&mut self)

    // Scans the next token from the source
    fn scan_token(&mut self) -> Token<'a> {
        self.skip_white_spaces();

        self.start = self.current;

        let c = match self.advance() {
            Some(c) => c,
            None => return Token::new(TokenType::Eof, "", self.line),
        };

        match c {
            '(' => self.make_token(TokenType::OpenParen),
            ')' => self.make_token(TokenType::CloseParen),
            '[' => self.make_token(TokenType::OpenBracket),
            ']' => self.make_token(TokenType::CloseBracket),
            '{' => self.make_token(TokenType::OpenBrace),
            '}' => self.make_token(TokenType::CloseBrace),
            ';' => self.make_token(TokenType::Semicolon),
            ',' => self.make_token(TokenType::Comma),
            '@' => self.make_token(TokenType::At),
            '$' => self.make_token(TokenType::Dollar),
            '?' => self.make_token(TokenType::Question),
            '_' => {
                // Check if the next character is not alphanumeric or another underscore
                if !matches!(self.peek(), Some(c) if c.is_alphanumeric() || c == '_') {
                    self.make_token(TokenType::Underscore)
                } else {
                    // Otherwise, scan it as an identifier
                    self.scan_identifier()
                }
            }
            '.' => {
                let kind = if self.peek() == Some('.') {
                    self.advance();
                    if self.peek() == Some('=') {
                        self.advance();
                        TokenType::DotDotEq
                    } else {
                        TokenType::DotDot
                    }
                } else {
                    TokenType::Dot
                };
                self.make_token(kind)
            }
            '|' => {
                let kind = if self.peek() == Some('>') {
                    self.advance();
                    TokenType::PipeArrow
                } else {
                    TokenType::Pipe
                };
                self.make_token(kind)
            }
            '-' => {
                let p = self.peek();
                let kind = if p == Some('>') {
                    self.advance();