Collapse runs of horizontal whitespace; keep paragraph breaks. Preserves single and double newlines (the markdown-paragraph signal) but collapses 3+ consecutive newlines to 2 so excessive blank gaps from PDF extraction disappear.
(text: str)
| 53 | |
| 54 | |
| 55 | def collapse_whitespace(text: str) -> str: |
| 56 | """Collapse runs of horizontal whitespace; keep paragraph breaks. |
| 57 | |
| 58 | Preserves single and double newlines (the markdown-paragraph signal) |
| 59 | but collapses 3+ consecutive newlines to 2 so excessive blank gaps from |
| 60 | PDF extraction disappear. |
| 61 | """ |
| 62 | if not text: |
| 63 | return "" |
| 64 | collapsed = _HORIZONTAL_WS_RE.sub(" ", text) |
| 65 | collapsed = _TRIPLE_NEWLINE_RE.sub("\n\n", collapsed) |
| 66 | # Strip trailing whitespace on each line without touching empties. |
| 67 | lines = [line.rstrip() for line in collapsed.split("\n")] |
| 68 | return "\n".join(lines).strip() |
| 69 | |
| 70 | |
| 71 | def dedupe_lines(text: str) -> str: |
no outgoing calls