A generator that yields the tokens in the given string.
(s)
| 380 | |
| 381 | |
| 382 | def Tokenize(s): |
| 383 | """A generator that yields the tokens in the given string.""" |
| 384 | if s != '': |
| 385 | lines = s.splitlines(True) |
| 386 | for token in TokenizeLines(lines, Cursor(0, 0)): |
| 387 | yield token |
| 388 | |
| 389 | |
| 390 | class CodeNode: |
no test coverage detected