Joins paragraphs that were prematurely split across lines (often due to PDF page/column breaks). Looks for paragraphs ending without terminal punctuation (e.g. ends with lowercase, comma, colon) and where the next paragraph begins with a lowercase letter.
(content: str)
| 573 | return '\n'.join(out), modified |
| 574 | |
| 575 | def join_split_paragraphs(content: str) -> tuple[str, int]: |
| 576 | """ |
| 577 | Joins paragraphs that were prematurely split across lines (often due to PDF page/column breaks). |
| 578 | Looks for paragraphs ending without terminal punctuation (e.g. ends with lowercase, comma, colon) |
| 579 | and where the next paragraph begins with a lowercase letter. |
| 580 | """ |
| 581 | import re |
| 582 | |
| 583 | # 1. Hyphenated words split across lines |
| 584 | # e.g. 'cyber-\n\nsecurity' -> 'cybersecurity' |
| 585 | content, n1 = re.subn(r'([a-zA-Z])-\s*\n+\s*([a-z])', r'\1\2', content) |
| 586 | |
| 587 | # 2. Regular line splits |
| 588 | # e.g. 'some sentence,\n\ncontinuing here.' -> 'some sentence, continuing here.' |
| 589 | content, n2 = re.subn(r'([a-z0-9:,;\)])\s*\n+\s*([a-z])', r'\1 \2', content) |
| 590 | |
| 591 | return content, n1 + n2 |
| 592 | |
| 593 | def apply_specific_fixes(content: str) -> tuple[str, int]: |
| 594 | """ |