Function tokenize

nodedb-sql/src/functions/fts_ops/tsquery_parser.rs:64–99

(input: &str)

// ── Tokenizer ────────────────────────────────────────────────────────────────

fn tokenize(input: &str) -> Vec<String> {
    let mut tokens = Vec::new();
    let mut chars = input.char_indices().peekable();
    while let Some((i, c)) = chars.next() {
        match c {
            ' ' | '\t' | '\n' | '\r' => continue,
            '(' | ')' | '&' | '|' | '!' => tokens.push(c.to_string()),
            ':' => {
                // ':*' suffix on the preceding term — already handled in term
                // parsing; if we see a bare ':' here it's part of the next
                // term text.  Push ':' as a token so we can detect ':*'.
                if chars.peek().map(|(_, nc)| *nc) == Some('*') {
                    chars.next(); // consume '*'
                    tokens.push(":*".into());
                } else {
                    tokens.push(":".into());
                }
            }
            _ => {
                // Collect a term: alphanumeric + hyphen + underscore.
                let start = i;
                let mut end = i + c.len_utf8();
                while let Some(&(j, nc)) = chars.peek() {
                    if nc.is_alphanumeric() || nc == '-' || nc == '_' {
                        end = j + nc.len_utf8();
                        chars.next();
                    } else {
                        break;
                    }
                }
                tokens.push(input[start..end].to_ascii_lowercase());
            }
        }
    }
    tokens
}

// ── Recursive-descent parser ──────────────────────────────────────────────────