| 240 | } |
| 241 | |
| 242 | fn lex_ident(bytes: &[u8], start: usize) -> (Token, usize) { |
| 243 | let mut i = start; |
| 244 | while i < bytes.len() |
| 245 | && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_' || bytes[i] == b':') |
| 246 | { |
| 247 | i += 1; |
| 248 | } |
| 249 | let word = std::str::from_utf8(&bytes[start..i]).unwrap_or(""); |
| 250 | |
| 251 | // Special identifier-like tokens. |
| 252 | match word { |
| 253 | "Inf" | "inf" => (Token::Number(f64::INFINITY), i), |
| 254 | "NaN" | "nan" => (Token::Number(f64::NAN), i), |
| 255 | _ => (Token::Ident(word.to_string()), i), |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | #[cfg(test)] |
| 260 | mod tests { |