r""" Takes a lexed list of tokens, and removes whitespace when there is a directive on a line by itself: >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) >>> tokens [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] >>> trim_lex(tokens)
(tokens)
| 704 | |
| 705 | |
| 706 | def trim_lex(tokens): |
| 707 | r""" |
| 708 | Takes a lexed list of tokens, and removes whitespace when there is |
| 709 | a directive on a line by itself: |
| 710 | |
| 711 | >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) |
| 712 | >>> tokens |
| 713 | [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] |
| 714 | >>> trim_lex(tokens) |
| 715 | [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y'] |
| 716 | """ |
| 717 | last_trim = None |
| 718 | for i in range(len(tokens)): |
| 719 | current = tokens[i] |
| 720 | if isinstance(tokens[i], basestring_): |
| 721 | # we don't trim this |
| 722 | continue |
| 723 | item = current[0] |
| 724 | if not statement_re.search(item) and item not in single_statements: |
| 725 | continue |
| 726 | if not i: |
| 727 | prev = '' |
| 728 | else: |
| 729 | prev = tokens[i - 1] |
| 730 | if i + 1 >= len(tokens): |
| 731 | next_chunk = '' |
| 732 | else: |
| 733 | next_chunk = tokens[i + 1] |
| 734 | if (not |
| 735 | isinstance(next_chunk, basestring_) or |
| 736 | not isinstance(prev, basestring_)): |
| 737 | continue |
| 738 | prev_ok = not prev or trail_whitespace_re.search(prev) |
| 739 | if i == 1 and not prev.strip(): |
| 740 | prev_ok = True |
| 741 | if last_trim is not None and last_trim + 2 == i and not prev.strip(): |
| 742 | prev_ok = 'last' |
| 743 | if (prev_ok and (not next_chunk or lead_whitespace_re.search( |
| 744 | next_chunk) or ( |
| 745 | i == len(tokens) - 2 and not next_chunk.strip()))): |
| 746 | if prev: |
| 747 | if ((i == 1 and not prev.strip()) or prev_ok == 'last'): |
| 748 | tokens[i - 1] = '' |
| 749 | else: |
| 750 | m = trail_whitespace_re.search(prev) |
| 751 | # +1 to leave the leading \n on: |
| 752 | prev = prev[:m.start() + 1] |
| 753 | tokens[i - 1] = prev |
| 754 | if next_chunk: |
| 755 | last_trim = i |
| 756 | if i == len(tokens) - 2 and not next_chunk.strip(): |
| 757 | tokens[i + 1] = '' |
| 758 | else: |
| 759 | m = lead_whitespace_re.search(next_chunk) |
| 760 | next_chunk = next_chunk[m.end():] |
| 761 | tokens[i + 1] = next_chunk |
| 762 | return tokens |
| 763 |