Function tokenize

nodedb-sql/src/ddl_ast/graph_parse/tokenizer.rs:12–40

(sql: &str)

}

pub(super) fn tokenize(sql: &str) -> Vec<Tok<'_>> {
    let bytes = sql.as_bytes();
    let mut out = Vec::new();
    let mut i = 0;
    while i < bytes.len() {
        let b = bytes[i];
        if b.is_ascii_whitespace()
            || b == b','
            || b == b';'
            || b == b'('
            || b == b')'
            || b == b'['
            || b == b']'
        {
            i += 1;
            continue;
        }
        if b == b'\'' {
            i = consume_quoted(sql, bytes, i, &mut out);
            continue;
        }
        if b == b'{' {
            i = consume_object(sql, bytes, i, &mut out);
            continue;
        }
        i = consume_word(sql, bytes, i, &mut out);
    }
    out
}

fn consume_quoted<'a>(sql: &'a str, bytes: &[u8], start: usize, out: &mut Vec<Tok<'a>>) -> usize {
    let content_start = start + 1;