Tokenize a SQL expression string into a list of [`Token`]s. This implementation iterates over `char`s (via `char_indices`) so that multi-byte UTF-8 codepoints are handled correctly. Slicing always uses byte offsets returned by `char_indices`, which are guaranteed to be on char boundaries.
(input: &str)
| 33 | /// byte offsets returned by `char_indices`, which are guaranteed to be on |
| 34 | /// char boundaries. |
| 35 | pub fn tokenize(input: &str) -> Result<Vec<Token>, ExprParseError> { |
| 36 | let chars: Vec<(usize, char)> = input.char_indices().collect(); |
| 37 | let mut tokens = Vec::new(); |
| 38 | let mut i = 0; |
| 39 | |
| 40 | while i < chars.len() { |
| 41 | let (_, ch) = chars[i]; |
| 42 | |
| 43 | // Skip whitespace. |
| 44 | if ch.is_ascii_whitespace() { |
| 45 | i += 1; |
| 46 | continue; |
| 47 | } |
| 48 | |
| 49 | // Single-char structural tokens. |
| 50 | if ch == '(' { |
| 51 | tokens.push(Token { |
| 52 | text: "(".into(), |
| 53 | kind: TokenKind::LParen, |
| 54 | }); |
| 55 | i += 1; |
| 56 | continue; |
| 57 | } |
| 58 | if ch == ')' { |
| 59 | tokens.push(Token { |
| 60 | text: ")".into(), |
| 61 | kind: TokenKind::RParen, |
| 62 | }); |
| 63 | i += 1; |
| 64 | continue; |
| 65 | } |
| 66 | if ch == ',' { |
| 67 | tokens.push(Token { |
| 68 | text: ",".into(), |
| 69 | kind: TokenKind::Comma, |
| 70 | }); |
| 71 | i += 1; |
| 72 | continue; |
| 73 | } |
| 74 | |
| 75 | // Two-char operators: <=, >=, !=, <>, || |
| 76 | if i + 1 < chars.len() { |
| 77 | let (_, next_ch) = chars[i + 1]; |
| 78 | let two: String = [ch, next_ch].iter().collect(); |
| 79 | if matches!(two.as_str(), "<=" | ">=" | "!=" | "<>" | "||") { |
| 80 | tokens.push(Token { |
| 81 | text: two, |
| 82 | kind: TokenKind::Op, |
| 83 | }); |
| 84 | i += 2; |
| 85 | continue; |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | // Single-char operators. |
| 90 | if matches!(ch, '+' | '-' | '*' | '/' | '%' | '=' | '<' | '>') { |
| 91 | tokens.push(Token { |
| 92 | text: ch.to_string(), |