Utility wrapper over python tokenizer.
| 492 | |
| 493 | |
| 494 | class PythonTokenizer: |
| 495 | """Utility wrapper over python tokenizer.""" |
| 496 | |
| 497 | def __init__(self, text): |
| 498 | self.text = text |
| 499 | i = iter([text]) |
| 500 | readline = lambda: next(i) |
| 501 | self.tokens = tokenize.generate_tokens(readline) |
| 502 | self.index = 0 |
| 503 | |
| 504 | def consume_till(self, delim): |
| 505 | """Consumes tokens till colon. |
| 506 | |
| 507 | >>> tok = PythonTokenizer('for i in range(10): hello $i') |
| 508 | >>> tok.consume_till(':') |
| 509 | >>> tok.text[:tok.index] |
| 510 | 'for i in range(10):' |
| 511 | >>> tok.text[tok.index:] |
| 512 | ' hello $i' |
| 513 | """ |
| 514 | try: |
| 515 | while True: |
| 516 | t = next(self) |
| 517 | if t.value == delim: |
| 518 | break |
| 519 | elif t.value == "(": |
| 520 | self.consume_till(")") |
| 521 | elif t.value == "[": |
| 522 | self.consume_till("]") |
| 523 | elif t.value == "{": |
| 524 | self.consume_till("}") |
| 525 | |
| 526 | # if end of line is found, it is an exception. |
| 527 | # Since there is no easy way to report the line number, |
| 528 | # leave the error reporting to the python parser later |
| 529 | # @@ This should be fixed. |
| 530 | if t.value == "\n": |
| 531 | break |
| 532 | except: |
| 533 | # raise ParseError, "Expected %s, found end of line." % repr(delim) |
| 534 | |
| 535 | # raising ParseError doesn't show the line number. |
| 536 | # if this error is ignored, then it will be caught when compiling the python code. |
| 537 | return |
| 538 | |
| 539 | def __next__(self): |
| 540 | type, t, begin, end, line = next(self.tokens) |
| 541 | row, col = end |
| 542 | self.index = col |
| 543 | return storage(type=type, value=t, begin=begin, end=end) |
| 544 | |
| 545 | |
| 546 | class DefwithNode: |
no outgoing calls
no test coverage detected