Token class.
| 230 | |
| 231 | |
| 232 | class Token(tuple): |
| 233 | """Token class.""" |
| 234 | __slots__ = () |
| 235 | lineno, type, value = (property(itemgetter(x)) for x in range(3)) |
| 236 | |
| 237 | def __new__(cls, lineno, type, value): |
| 238 | return tuple.__new__(cls, (lineno, intern(str(type)), value)) |
| 239 | |
| 240 | def __str__(self): |
| 241 | if self.type in reverse_operators: |
| 242 | return reverse_operators[self.type] |
| 243 | elif self.type == 'name': |
| 244 | return self.value |
| 245 | return self.type |
| 246 | |
| 247 | def test(self, expr): |
| 248 | """Test a token against a token expression. This can either be a |
| 249 | token type or ``'token_type:token_value'``. This can only test |
| 250 | against string values and types. |
| 251 | """ |
| 252 | # here we do a regular string equality check as test_any is usually |
| 253 | # passed an iterable of not interned strings. |
| 254 | if self.type == expr: |
| 255 | return True |
| 256 | elif ':' in expr: |
| 257 | return expr.split(':', 1) == [self.type, self.value] |
| 258 | return False |
| 259 | |
| 260 | def test_any(self, *iterable): |
| 261 | """Test against multiple token expressions.""" |
| 262 | for expr in iterable: |
| 263 | if self.test(expr): |
| 264 | return True |
| 265 | return False |
| 266 | |
| 267 | def __repr__(self): |
| 268 | return 'Token(%r, %r, %r)' % ( |
| 269 | self.lineno, |
| 270 | self.type, |
| 271 | self.value |
| 272 | ) |
| 273 | |
| 274 | |
| 275 | @implements_iterator |