Tokenize the given input using the configured commandset.
(self, commands: io.TextIOWrapper)
| 45 | self.stack = [] |
| 46 | |
| 47 | def tokenize(self, commands: io.TextIOWrapper) -> Tokens: |
| 48 | """Tokenize the given input using the configured commandset.""" |
| 49 | if self.commandset == "default": |
| 50 | return self._tokenize_default(commands) |
| 51 | raise ValueError(f"tokenize not supported (yet) for '{self.commandset}'") |
| 52 | |
| 53 | @staticmethod |
| 54 | def _tokenize_default(commands: io.TextIOWrapper) -> Tokens: |