Context-sensitive token parsing. Provides methods to examine the input stream to check whether the next token matches.
| 86 | |
| 87 | |
| 88 | class Tokenizer: |
| 89 | """Context-sensitive token parsing. |
| 90 | |
| 91 | Provides methods to examine the input stream to check whether the next token |
| 92 | matches. |
| 93 | """ |
| 94 | |
| 95 | def __init__( |
| 96 | self, |
| 97 | source: str, |
| 98 | *, |
| 99 | rules: "Dict[str, Union[str, re.Pattern[str]]]", |
| 100 | ) -> None: |
| 101 | self.source = source |
| 102 | self.rules: Dict[str, re.Pattern[str]] = { |
| 103 | name: re.compile(pattern) for name, pattern in rules.items() |
| 104 | } |
| 105 | self.next_token: Optional[Token] = None |
| 106 | self.position = 0 |
| 107 | |
| 108 | def consume(self, name: str) -> None: |
| 109 | """Move beyond provided token name, if at current position.""" |
| 110 | if self.check(name): |
| 111 | self.read() |
| 112 | |
| 113 | def check(self, name: str, *, peek: bool = False) -> bool: |
| 114 | """Check whether the next token has the provided name. |
| 115 | |
| 116 | By default, if the check succeeds, the token *must* be read before |
| 117 | another check. If `peek` is set to `True`, the token is not loaded and |
| 118 | would need to be checked again. |
| 119 | """ |
| 120 | assert ( |
| 121 | self.next_token is None |
| 122 | ), f"Cannot check for {name!r}, already have {self.next_token!r}" |
| 123 | assert name in self.rules, f"Unknown token name: {name!r}" |
| 124 | |
| 125 | expression = self.rules[name] |
| 126 | |
| 127 | match = expression.match(self.source, self.position) |
| 128 | if match is None: |
| 129 | return False |
| 130 | if not peek: |
| 131 | self.next_token = Token(name, match[0], self.position) |
| 132 | return True |
| 133 | |
| 134 | def expect(self, name: str, *, expected: str) -> Token: |
| 135 | """Expect a certain token name next, failing with a syntax error otherwise. |
| 136 | |
| 137 | The token is *not* read. |
| 138 | """ |
| 139 | if not self.check(name): |
| 140 | raise self.raise_syntax_error(f"Expected {expected}") |
| 141 | return self.read() |
| 142 | |
| 143 | def read(self) -> Token: |
| 144 | """Consume the next token and return it.""" |
| 145 | token = self.next_token |
no outgoing calls
no test coverage detected