This method tokenizes the text and returns the tokens in a generator. Use this method if you just want to tokenize a template.
(self, source, name, filename=None, state=None)
| 597 | yield Token(lineno, token, value) |
| 598 | |
| 599 | def tokeniter(self, source, name, filename=None, state=None): |
| 600 | """This method tokenizes the text and returns the tokens in a |
| 601 | generator. Use this method if you just want to tokenize a template. |
| 602 | """ |
| 603 | source = text_type(source) |
| 604 | lines = source.splitlines() |
| 605 | if self.keep_trailing_newline and source: |
| 606 | for newline in ('\r\n', '\r', '\n'): |
| 607 | if source.endswith(newline): |
| 608 | lines.append('') |
| 609 | break |
| 610 | source = '\n'.join(lines) |
| 611 | pos = 0 |
| 612 | lineno = 1 |
| 613 | stack = ['root'] |
| 614 | if state is not None and state != 'root': |
| 615 | assert state in ('variable', 'block'), 'invalid state' |
| 616 | stack.append(state + '_begin') |
| 617 | else: |
| 618 | state = 'root' |
| 619 | statetokens = self.rules[stack[-1]] |
| 620 | source_length = len(source) |
| 621 | |
| 622 | balancing_stack = [] |
| 623 | |
| 624 | while 1: |
| 625 | # tokenizer loop |
| 626 | for regex, tokens, new_state in statetokens: |
| 627 | m = regex.match(source, pos) |
| 628 | # if no match we try again with the next rule |
| 629 | if m is None: |
| 630 | continue |
| 631 | |
| 632 | # we only match blocks and variables if braces / parentheses |
| 633 | # are balanced. continue parsing with the lower rule which |
| 634 | # is the operator rule. do this only if the end tags look |
| 635 | # like operators |
| 636 | if balancing_stack and \ |
| 637 | tokens in ('variable_end', 'block_end', |
| 638 | 'linestatement_end'): |
| 639 | continue |
| 640 | |
| 641 | # tuples support more options |
| 642 | if isinstance(tokens, tuple): |
| 643 | for idx, token in enumerate(tokens): |
| 644 | # failure group |
| 645 | if token.__class__ is Failure: |
| 646 | raise token(lineno, filename) |
| 647 | # bygroup is a bit more complex, in that case we |
| 648 | # yield for the current token the first named |
| 649 | # group that matched |
| 650 | elif token == '#bygroup': |
| 651 | for key, value in iteritems(m.groupdict()): |
| 652 | if value is not None: |
| 653 | yield lineno, key, value |
| 654 | lineno += value.count('\n') |
| 655 | break |
| 656 | else: |