Drop consecutive duplicate lines. Useful for killing repeated PDF page headers/footers ("Page 3 of 12", journal banners) without touching legitimate prose. Comparison is whitespace-insensitive — leading/trailing space on either copy still counts as a duplicate.
(text: str)
| 69 | |
| 70 | |
| 71 | def dedupe_lines(text: str) -> str: |
| 72 | """Drop consecutive duplicate lines. |
| 73 | |
| 74 | Useful for killing repeated PDF page headers/footers ("Page 3 of 12", |
| 75 | journal banners) without touching legitimate prose. Comparison is |
| 76 | whitespace-insensitive — leading/trailing space on either copy still |
| 77 | counts as a duplicate. |
| 78 | """ |
| 79 | if not text: |
| 80 | return "" |
| 81 | output: list[str] = [] |
| 82 | last: str | None = None |
| 83 | for line in text.split("\n"): |
| 84 | key = line.strip() |
| 85 | if key and key == last: |
| 86 | continue |
| 87 | output.append(line) |
| 88 | last = key |
| 89 | return "\n".join(output) |
no outgoing calls