(&mut self)
| 156 | } |
| 157 | |
| 158 | fn lex_token(&mut self) -> TokenKind { |
| 159 | if let Some(fstring) = self.fstrings.current() { |
| 160 | if !fstring.is_in_expression(self.nesting) { |
| 161 | if let Some(token) = self.lex_fstring_middle_or_end() { |
| 162 | if matches!(token, TokenKind::FStringEnd) { |
| 163 | self.fstrings.pop(); |
| 164 | } |
| 165 | return token; |
| 166 | } |
| 167 | } |
| 168 | } |
| 169 | // Return dedent tokens until the current indentation level matches the indentation of the next token. |
| 170 | else if let Some(indentation) = self.pending_indentation.take() { |
| 171 | match self.indentations.current().try_compare(indentation) { |
| 172 | Ok(Ordering::Greater) => { |
| 173 | self.pending_indentation = Some(indentation); |
| 174 | if self.indentations.dedent_one(indentation).is_err() { |
| 175 | return self.push_error(LexicalError::new( |
| 176 | LexicalErrorType::IndentationError, |
| 177 | self.token_range(), |
| 178 | )); |
| 179 | } |
| 180 | return TokenKind::Dedent; |
| 181 | } |
| 182 | Ok(_) => {} |
| 183 | Err(_) => { |
| 184 | return self.push_error(LexicalError::new( |
| 185 | LexicalErrorType::IndentationError, |
| 186 | self.token_range(), |
| 187 | )); |
| 188 | } |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | if self.state.is_after_newline() { |
| 193 | if let Some(indentation) = self.eat_indentation() { |
| 194 | return indentation; |
| 195 | } |
| 196 | } else if let Err(error) = self.skip_whitespace() { |
| 197 | return self.push_error(error); |
| 198 | } |
| 199 | |
| 200 | // The lexer might've skipped whitespaces, so update the start offset |
| 201 | self.cursor.start_token(); |
| 202 | |
| 203 | if let Some(c) = self.cursor.bump() { |
| 204 | if c.is_ascii() { |
| 205 | self.consume_ascii_character(c) |
| 206 | } else if is_unicode_identifier_start(c) { |
| 207 | let identifier = self.lex_identifier(c); |
| 208 | self.state = State::Other; |
| 209 | |
| 210 | identifier |
| 211 | } else { |
| 212 | self.push_error(LexicalError::new( |
| 213 | LexicalErrorType::UnrecognizedToken { tok: c }, |
| 214 | self.token_range(), |
| 215 | )) |
no test coverage detected