| 309 | } |
| 310 | |
| 311 | pub fn tokenize(&mut self) -> Result<Vec<Token>, String> { |
| 312 | let mut remaining = self.input.as_str(); |
| 313 | let mut tokens = Vec::new(); |
| 314 | let mut iteration = 0; |
| 315 | |
| 316 | // Main tokenization loop - processes input string character by character |
| 317 | while !remaining.is_empty() { |
| 318 | iteration += 1; |
| 319 | |
| 320 | // CRITICAL: Infinite loop protection |
| 321 | // This prevents the lexer from getting stuck when no progress is made |
| 322 | // Common causes of infinite loops: |
| 323 | // 1. Whitespace function returning Token::Whitespace without consuming input |
| 324 | // 2. Function parsers not advancing the input position |
| 325 | // 3. Parser functions returning the same remaining string |
| 326 | if iteration > 1000 { |
| 327 | return Err("Infinite loop detected in lexer".to_string()); |
| 328 | } |
| 329 | |
| 330 | match token(remaining) { |
| 331 | Ok((next_remaining, token)) => { |
| 332 | // CRITICAL: Ensure input position advances |
| 333 | // If next_remaining == remaining, we have an infinite loop |
| 334 | // This check helps debug parser functions that don't consume input |
| 335 | if next_remaining == remaining { |
| 336 | return Err(format!( |
| 337 | "Parser function not consuming input. Token: {:?}, Remaining: '{}'", |
| 338 | token, remaining |
| 339 | )); |
| 340 | } |
| 341 | |
| 342 | // Only add non-whitespace/non-comment tokens to the result |
| 343 | if !matches!(token, Token::Whitespace | Token::Comment(_)) { |
| 344 | tokens.push(token); |
| 345 | } |
| 346 | remaining = next_remaining; |
| 347 | } |
| 348 | Err(e) => { |
| 349 | return Err(format!("Lexer error: {:?}", e)); |
| 350 | } |
| 351 | } |
| 352 | } |
| 353 | tokens.push(Token::Eof); |
| 354 | self.tokens = tokens.clone(); |
| 355 | Ok(tokens) |
| 356 | } |
| 357 | } |
| 358 | |
| 359 | /// Parse a single token using hybrid approach |