r""" Takes a lexed set of tokens, and removes whitespace when there is a directive on a line by itself: >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) >>> tokens [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] >>> trim_lex(tokens)
(tokens)
| 634 | |
| 635 | |
| 636 | def trim_lex(tokens): |
| 637 | r""" |
| 638 | Takes a lexed set of tokens, and removes whitespace when there is |
| 639 | a directive on a line by itself: |
| 640 | |
| 641 | >>> tokens = lex('{{if x}}\nx\n{{endif}}\ny', trim_whitespace=False) |
| 642 | >>> tokens |
| 643 | [('if x', (1, 3)), '\nx\n', ('endif', (3, 3)), '\ny'] |
| 644 | >>> trim_lex(tokens) |
| 645 | [('if x', (1, 3)), 'x\n', ('endif', (3, 3)), 'y'] |
| 646 | """ |
| 647 | last_trim = None |
| 648 | for i, current in enumerate(tokens): |
| 649 | if isinstance(current, basestring_): |
| 650 | # we don't trim this |
| 651 | continue |
| 652 | item = current[0] |
| 653 | if not statement_re.search(item) and item not in single_statements: |
| 654 | continue |
| 655 | if not i: |
| 656 | prev = "" |
| 657 | else: |
| 658 | prev = tokens[i - 1] |
| 659 | if i + 1 >= len(tokens): |
| 660 | next_chunk = "" |
| 661 | else: |
| 662 | next_chunk = tokens[i + 1] |
| 663 | if not isinstance(next_chunk, basestring_) or not isinstance(prev, basestring_): |
| 664 | continue |
| 665 | prev_ok = not prev or trail_whitespace_re.search(prev) |
| 666 | if i == 1 and not prev.strip(): |
| 667 | prev_ok = True |
| 668 | if last_trim is not None and last_trim + 2 == i and not prev.strip(): |
| 669 | prev_ok = "last" |
| 670 | if prev_ok and ( |
| 671 | not next_chunk |
| 672 | or lead_whitespace_re.search(next_chunk) |
| 673 | or (i == len(tokens) - 2 and not next_chunk.strip()) |
| 674 | ): |
| 675 | if prev: |
| 676 | if (i == 1 and not prev.strip()) or prev_ok == "last": |
| 677 | tokens[i - 1] = "" |
| 678 | else: |
| 679 | m = trail_whitespace_re.search(prev) |
| 680 | # +1 to leave the leading \n on: |
| 681 | prev = prev[: m.start() + 1] |
| 682 | tokens[i - 1] = prev |
| 683 | if next_chunk: |
| 684 | last_trim = i |
| 685 | if i == len(tokens) - 2 and not next_chunk.strip(): |
| 686 | tokens[i + 1] = "" |
| 687 | else: |
| 688 | m = lead_whitespace_re.search(next_chunk) |
| 689 | next_chunk = next_chunk[m.end():] |
| 690 | tokens[i + 1] = next_chunk |
| 691 | return tokens |
| 692 | |
| 693 |