| 390 | |
| 391 | #[allow(clippy::should_implement_trait)] |
| 392 | pub fn next(&mut self) -> Cursor { |
| 393 | // Collect trivia that should be associated with the next content token |
| 394 | let mut pending_trivia = Vec::new_in(self.bump); |
| 395 | |
| 396 | loop { |
| 397 | if self.buffer_index >= BUFFER_REFILL_INDEX { |
| 398 | self.fill_buffer(self.buffer_index); |
| 399 | } |
| 400 | |
| 401 | for i in self.buffer_index..BUFFER_LEN { |
| 402 | let c = self.buffer[i]; |
| 403 | if c == Kind::Eof { |
| 404 | self.buffer_index = i; |
| 405 | // Associate pending trivia with EOF if any |
| 406 | if !pending_trivia.is_empty() { |
| 407 | self.trivia.push((pending_trivia.clone(), c)); |
| 408 | } |
| 409 | #[cfg(debug_assertions)] |
| 410 | { |
| 411 | self.last_cursor = None; |
| 412 | } |
| 413 | return c; |
| 414 | } else if c == self.skip { |
| 415 | pending_trivia.push(c); |
| 416 | } else { |
| 417 | self.buffer_index = i + 1; |
| 418 | if self.buffer_index >= BUFFER_REFILL_INDEX { |
| 419 | self.fill_buffer(self.buffer_index); |
| 420 | } |
| 421 | // Associate all pending trivia with this content token |
| 422 | if !pending_trivia.is_empty() { |
| 423 | self.trivia.push((pending_trivia.clone(), c)); |
| 424 | } |
| 425 | #[cfg(debug_assertions)] |
| 426 | { |
| 427 | if let Some(last_cursor) = self.last_cursor { |
| 428 | debug_assert!(last_cursor != c, "Detected a next loop, {c:?} was fetched twice"); |
| 429 | } |
| 430 | self.last_cursor = Some(c); |
| 431 | } |
| 432 | return c; |
| 433 | } |
| 434 | } |
| 435 | |
| 436 | // Buffer exhausted with only skip tokens. Refill so buffer_index stays valid. |
| 437 | self.fill_buffer(BUFFER_LEN); |
| 438 | } |
| 439 | } |
| 440 | } |
| 441 | |
| 442 | #[test] |