docling often emits the document title/date 2-3 times at the top. Remove exact duplicate consecutive paragraphs (non-table, non-heading).
(content: str)
| 394 | # ── duplicate header deduplication ─────────────────────────────────────────── |
| 395 | |
| 396 | def deduplicate_headers(content: str) -> str: |
| 397 | """ |
| 398 | docling often emits the document title/date 2-3 times at the top. |
| 399 | Remove exact duplicate consecutive paragraphs (non-table, non-heading). |
| 400 | """ |
| 401 | lines = content.split('\n') |
| 402 | seen_paragraphs: set[str] = set() |
| 403 | out = [] |
| 404 | i = 0 |
| 405 | while i < len(lines): |
| 406 | line = lines[i] |
| 407 | # Collect a paragraph (lines until blank) |
| 408 | if line.strip() and not line.startswith('#') and not line.startswith('|') \ |
| 409 | and not line.startswith('>') and not line.startswith('-') \ |
| 410 | and not line.startswith('*') and not line.startswith('['): |
| 411 | para_lines = [] |
| 412 | j = i |
| 413 | while j < len(lines) and lines[j].strip(): |
| 414 | para_lines.append(lines[j]) |
| 415 | j += 1 |
| 416 | para_text = ' '.join(para_lines).strip() |
| 417 | # Only deduplicate short paragraphs (likely repeated titles/dates) |
| 418 | if len(para_text) < 300 and para_text in seen_paragraphs: |
| 419 | i = j # skip this duplicate paragraph |
| 420 | continue |
| 421 | seen_paragraphs.add(para_text) |
| 422 | out.append(line) |
| 423 | i += 1 |
| 424 | return '\n'.join(out) |
| 425 | |
| 426 | |
| 427 | # ── collapse excess blank lines ─────────────────────────────────────────────── |