Method tokenize_characters

crates/gitql-parser/src/tokenizer.rs:46–413

(&mut self)

    }

    fn tokenize_characters(&mut self) -> Result<Vec<Token>, Box<Diagnostic>> {
        let mut tokens: Vec<Token> = Vec::new();
        let len = self.content_len;

        while self.has_next() {
            self.column_start = self.column_end;
            self.line_start = self.line_end;

            let char = self.content[self.index];

            // Symbol
            if char.is_alphabetic() {
                tokens.push(self.consume_identifier());
                continue;
            }

            // @> or Global Variable Symbol
            if char == '@' {
                // @>
                if self.is_next_char('>') {
                    self.index += 2;
                    let location = self.current_source_location();
                    tokens.push(Token::new(TokenKind::AtRightArrow, location));
                    continue;
                }

                tokens.push(self.consume_global_variable_name()?);
                continue;
            }

            // Number
            if char.is_numeric() {
                if char == '0' && self.index + 1 < len {
                    match self.content[self.index + 1] {
                        // bindigits
                        'b' | 'B' => {
                            self.index += 2;
                            self.column_start += 2;
                            tokens.push(self.consume_binary_number()?);
                            continue;
                        }
                        // hexdigits
                        'x' | 'X' => {
                            self.index += 2;
                            self.column_start += 2;
                            tokens.push(self.consume_hex_number()?);
                            continue;
                        }
                        // octdigits
                        'o' | 'O' => {
                            self.index += 2;
                            self.column_start += 2;
                            tokens.push(self.consume_octal_number()?);
                            continue;
                        }
                        _ => {
                            tokens.push(self.consume_number()?);
                            continue;