(pages: list[dict[str, Any]], repeated_margin_lines: set[str])
| 125 | |
| 126 | |
| 127 | def clean_pages(pages: list[dict[str, Any]], repeated_margin_lines: set[str]) -> list[dict[str, Any]]: |
| 128 | cleaned: list[dict[str, Any]] = [] |
| 129 | for p in pages: |
| 130 | kept_lines: list[str] = [] |
| 131 | for line in p["raw_text"].splitlines(): |
| 132 | s = normalize_line(line) |
| 133 | if not s: |
| 134 | kept_lines.append("") |
| 135 | continue |
| 136 | if s in repeated_margin_lines: |
| 137 | continue |
| 138 | if RE_PAGE_ONLY.match(s): |
| 139 | continue |
| 140 | kept_lines.append(s) |
| 141 | |
| 142 | text = normalize_text("\n".join(kept_lines)) |
| 143 | cleaned.append({"page": p["page"], "text": text, "lines": kept_lines}) |
| 144 | return cleaned |
| 145 | |
| 146 | |
| 147 | def classify_heading(line: str) -> dict[str, str] | None: |
no test coverage detected