A token stream is an iterable that yields :class:`Token`\\s. The parser however does not iterate over it but calls :meth:`next` to go one token ahead. The current active token is stored as :attr:`current`.
| 295 | |
| 296 | @implements_iterator |
| 297 | class TokenStream(object): |
| 298 | """A token stream is an iterable that yields :class:`Token`\\s. The |
| 299 | parser however does not iterate over it but calls :meth:`next` to go |
| 300 | one token ahead. The current active token is stored as :attr:`current`. |
| 301 | """ |
| 302 | |
| 303 | def __init__(self, generator, name, filename): |
| 304 | self._iter = iter(generator) |
| 305 | self._pushed = deque() |
| 306 | self.name = name |
| 307 | self.filename = filename |
| 308 | self.closed = False |
| 309 | self.current = Token(1, TOKEN_INITIAL, '') |
| 310 | next(self) |
| 311 | |
| 312 | def __iter__(self): |
| 313 | return TokenStreamIterator(self) |
| 314 | |
| 315 | def __bool__(self): |
| 316 | return bool(self._pushed) or self.current.type is not TOKEN_EOF |
| 317 | __nonzero__ = __bool__ # py2 |
| 318 | |
| 319 | eos = property(lambda x: not x, doc="Are we at the end of the stream?") |
| 320 | |
| 321 | def push(self, token): |
| 322 | """Push a token back to the stream.""" |
| 323 | self._pushed.append(token) |
| 324 | |
| 325 | def look(self): |
| 326 | """Look at the next token.""" |
| 327 | old_token = next(self) |
| 328 | result = self.current |
| 329 | self.push(result) |
| 330 | self.current = old_token |
| 331 | return result |
| 332 | |
| 333 | def skip(self, n=1): |
| 334 | """Got n tokens ahead.""" |
| 335 | for x in range(n): |
| 336 | next(self) |
| 337 | |
| 338 | def next_if(self, expr): |
| 339 | """Perform the token test and return the token if it matched. |
| 340 | Otherwise the return value is `None`. |
| 341 | """ |
| 342 | if self.current.test(expr): |
| 343 | return next(self) |
| 344 | |
| 345 | def skip_if(self, expr): |
| 346 | """Like :meth:`next_if` but only returns `True` or `False`.""" |
| 347 | return self.next_if(expr) is not None |
| 348 | |
| 349 | def __next__(self): |
| 350 | """Go one token ahead and return the old one. |
| 351 | |
| 352 | Use the built-in :func:`next` instead of calling this directly. |
| 353 | """ |
| 354 | rv = self.current |