A token stream is an iterable that yields :class:`Token`\\s. The parser however does not iterate over it but calls :meth:`next` to go one token ahead. The current active token is stored as :attr:`current`.
| 315 | |
| 316 | |
| 317 | class TokenStream: |
| 318 | """A token stream is an iterable that yields :class:`Token`\\s. The |
| 319 | parser however does not iterate over it but calls :meth:`next` to go |
| 320 | one token ahead. The current active token is stored as :attr:`current`. |
| 321 | """ |
| 322 | |
| 323 | def __init__( |
| 324 | self, |
| 325 | generator: t.Iterable[Token], |
| 326 | name: t.Optional[str], |
| 327 | filename: t.Optional[str], |
| 328 | ): |
| 329 | self._iter = iter(generator) |
| 330 | self._pushed: "te.Deque[Token]" = deque() |
| 331 | self.name = name |
| 332 | self.filename = filename |
| 333 | self.closed = False |
| 334 | self.current = Token(1, TOKEN_INITIAL, "") |
| 335 | next(self) |
| 336 | |
| 337 | def __iter__(self) -> TokenStreamIterator: |
| 338 | return TokenStreamIterator(self) |
| 339 | |
| 340 | def __bool__(self) -> bool: |
| 341 | return bool(self._pushed) or self.current.type is not TOKEN_EOF |
| 342 | |
| 343 | @property |
| 344 | def eos(self) -> bool: |
| 345 | """Are we at the end of the stream?""" |
| 346 | return not self |
| 347 | |
| 348 | def push(self, token: Token) -> None: |
| 349 | """Push a token back to the stream.""" |
| 350 | self._pushed.append(token) |
| 351 | |
| 352 | def look(self) -> Token: |
| 353 | """Look at the next token.""" |
| 354 | old_token = next(self) |
| 355 | result = self.current |
| 356 | self.push(result) |
| 357 | self.current = old_token |
| 358 | return result |
| 359 | |
| 360 | def skip(self, n: int = 1) -> None: |
| 361 | """Got n tokens ahead.""" |
| 362 | for _ in range(n): |
| 363 | next(self) |
| 364 | |
| 365 | def next_if(self, expr: str) -> t.Optional[Token]: |
| 366 | """Perform the token test and return the token if it matched. |
| 367 | Otherwise the return value is `None`. |
| 368 | """ |
| 369 | if self.current.test(expr): |
| 370 | return next(self) |
| 371 | |
| 372 | return None |
| 373 | |
| 374 | def skip_if(self, expr: str) -> bool: |