| 130 | buffer = [] |
| 131 | |
| 132 | def send_buffer() -> Iterator[Token]: |
| 133 | nonlocal backslashes |
| 134 | if not buffer: |
| 135 | return None |
| 136 | |
| 137 | value = ''.join(buffer) |
| 138 | kind = TokenKind.TEXT |
| 139 | if not backslashes: |
| 140 | for variation, kind in [ |
| 141 | (int, TokenKind.NUMBER), |
| 142 | (check_escaped_int, TokenKind.TEXT), |
| 143 | ]: |
| 144 | try: |
| 145 | value = variation(value) |
| 146 | except ValueError: |
| 147 | continue |
| 148 | else: |
| 149 | break |
| 150 | yield Token( |
| 151 | kind=kind, |
| 152 | value=value, |
| 153 | start=cursor - (len(buffer) + backslashes), |
| 154 | end=cursor, |
| 155 | ) |
| 156 | buffer.clear() |
| 157 | backslashes = 0 |
| 158 | |
| 159 | def can_advance() -> bool: |
| 160 | return cursor < len(source) |