Function tokenize

nodedb-sql/src/parser/array_stmt/lexer.rs:40–218

Tokenize a SQL slice. Returns `Vec ` on success; any unexpected character produces `SqlError::Parse`.

(src: &str)

/// Tokenize a SQL slice. Returns `Vec<Token>` on success; any unexpected
/// character produces `SqlError::Parse`.
pub fn tokenize(src: &str) -> Result<Vec<Token>, SqlError> {
    let bytes = src.as_bytes();
    let mut out = Vec::with_capacity(src.len() / 4);
    let mut i = 0;
    while i < bytes.len() {
        let b = bytes[i];
        // Whitespace.
        if b.is_ascii_whitespace() {
            i += 1;
            continue;
        }
        // Single-line comment `-- ...`.
        if b == b'-' && i + 1 < bytes.len() && bytes[i + 1] == b'-' {
            while i < bytes.len() && bytes[i] != b'\n' {
                i += 1;
            }
            continue;
        }
        // Punctuation.
        match b {
            b'(' => {
                out.push(Token {
                    tok: Tok::LParen,
                    pos: i,
                });
                i += 1;
                continue;
            }
            b')' => {
                out.push(Token {
                    tok: Tok::RParen,
                    pos: i,
                });
                i += 1;
                continue;
            }
            b'[' => {
                out.push(Token {
                    tok: Tok::LBracket,
                    pos: i,
                });
                i += 1;
                continue;
            }
            b']' => {
                out.push(Token {
                    tok: Tok::RBracket,
                    pos: i,
                });
                i += 1;
                continue;
            }
            b',' => {
                out.push(Token {
                    tok: Tok::Comma,
                    pos: i,
                });
                i += 1;