Method read_next_token

crates/css_lexer/src/private.rs:900–1083

(&mut self, offset: u32)

impl<'a> Lexer<'a> {
	#[must_use]
	pub(crate) fn read_next_token(&mut self, offset: u32) -> Token {
		if self.source.len() as u32 == offset {
			return Token::EOF;
		}
		let bytes = self.source.as_bytes();
		let mut cursor = ByteCursor::new(&bytes[offset as usize..]);
		let b = cursor.peek_byte(0);
		// fast path for single character tokens (ASCII only)
		if b < 128 {
			let token = SINGLE_CHAR_TOKENS[b as usize];
			if token != Token::EOF {
				return token;
			}
			// fast path for identifiers
			if is_ident_ascii_start(b as char) {
				if matches!(b, b'U' | b'u')
					&& self.features.intersects(Feature::UnicodeRange)
					&& cursor.would_start_unicode_range()
				{
					return cursor.consume_unicode_range_token();
				}
				return cursor.consume_ident_like_token(self.atoms);
			}
		}
		let c = cursor.peek_char();
		match c {
			'\0' => {
				if !cursor.is_last() {
					let (_, c2, c3) = cursor.peek3();
					if is_ident_start_sequence(REPLACEMENT_CHARACTER, c2, c3) {
						return cursor.consume_ident_like_token(self.atoms);
					}
				}
				if cursor.next_char().is_some() { Token::REPLACEMENT_CHARACTER } else { Token::EOF }
			}
			c if is_whitespace(c) && !self.features.contains(Feature::SeparateWhitespace) => {
				let (len, style) = cursor.consume_whitespace();
				Token::new_whitespace(style, len)
			}
			// Whitespace Range
			TAB => Token::new_whitespace(Whitespace::Tab, cursor.consume_same(b'\t')),
			SPACE => Token::new_whitespace(Whitespace::Space, cursor.consume_same(b' ')),
			LF | CR | FF => {
				let mut len: u32 = 0;
				loop {
					let nb = cursor.peek_byte(0);
					if !matches!(nb, b'\n' | b'\r' | 0x0C) {
						break;
					}
					cursor.pos += 1;
					len += 1;
				}
				Token::new_whitespace(Whitespace::Newline, len)
			}
			// Quote Range
			c if is_quote(c) => cursor.consume_string_token(),
			// Digit Range
			c if c.is_ascii_digit() => cursor.consume_numeric_token(self.atoms),