Function tokenize

nodedb/src/control/planner/procedural/tokenizer.rs:81–222

Tokenize procedural SQL text into a token stream. The tokenizer is keyword-aware: it recognizes procedural keywords and captures everything else as `SqlFragment` tokens. String literals are preserved (not split on keywords inside strings).

(input: &str)

/// captures everything else as `SqlFragment` tokens. String literals are
/// preserved (not split on keywords inside strings).
pub fn tokenize(input: &str) -> Result<Vec<Token>, ProceduralError> {
    let mut tokens = Vec::new();
    let bytes = input.as_bytes();
    let len = bytes.len();
    let mut i = 0;

    while i < len {
        // Skip whitespace.
        if bytes[i].is_ascii_whitespace() {
            i += 1;
            continue;
        }

        // Skip SQL comments.
        if i + 1 < len && bytes[i] == b'-' && bytes[i + 1] == b'-' {
            while i < len && bytes[i] != b'\n' {
                i += 1;
            }
            continue;
        }

        // Semicolon.
        if bytes[i] == b';' {
            tokens.push(Token::Semicolon);
            i += 1;
            continue;
        }

        // `:=` assignment.
        if i + 1 < len && bytes[i] == b':' && bytes[i + 1] == b'=' {
            tokens.push(Token::Assign);
            i += 2;
            continue;
        }

        // `..` range operator.
        if i + 1 < len && bytes[i] == b'.' && bytes[i + 1] == b'.' {
            tokens.push(Token::DotDot);
            i += 2;
            continue;
        }

        // String literal.
        if bytes[i] == b'\'' {
            let (lit, end) = read_string_literal(input, i)?;
            tokens.push(Token::StringLit(lit));
            i = end;
            continue;
        }

        // Number literal.
        if bytes[i].is_ascii_digit() {
            let start = i;
            while i < len {
                if bytes[i].is_ascii_digit() {
                    i += 1;
                } else if bytes[i] == b'.' {
                    // Check for `..` (range operator) — don't consume the dot.