| 45 | del token |
| 46 | |
| 47 | class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): |
| 48 | def __repr__(self): |
| 49 | annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) |
| 50 | return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % |
| 51 | self._replace(type=annotated_type)) |
| 52 | |
| 53 | @property |
| 54 | def exact_type(self): |
| 55 | if self.type == OP and self.string in EXACT_TOKEN_TYPES: |
| 56 | return EXACT_TOKEN_TYPES[self.string] |
| 57 | else: |
| 58 | return self.type |
| 59 | |
| 60 | def group(*choices): return '(' + '|'.join(choices) + ')' |
| 61 | def any(*choices): return group(*choices) + '*' |