(pages: list[dict[str, Any]])
| 105 | |
| 106 | |
| 107 | def detect_repeated_margin_lines(pages: list[dict[str, Any]]) -> set[str]: |
| 108 | top_counter: Counter[str] = Counter() |
| 109 | bottom_counter: Counter[str] = Counter() |
| 110 | |
| 111 | for p in pages: |
| 112 | lines = [normalize_line(ln) for ln in p["raw_text"].splitlines() if ln.strip()] |
| 113 | if not lines: |
| 114 | continue |
| 115 | |
| 116 | top_lines = [ln for ln in lines[:3] if not is_probably_boilerplate(ln)] |
| 117 | bottom_lines = [ln for ln in lines[-3:] if not is_probably_boilerplate(ln)] |
| 118 | top_counter.update(top_lines) |
| 119 | bottom_counter.update(bottom_lines) |
| 120 | |
| 121 | threshold = max(3, math.ceil(len(pages) * 0.08)) |
| 122 | repeated = {line for line, c in top_counter.items() if c >= threshold} |
| 123 | repeated |= {line for line, c in bottom_counter.items() if c >= threshold} |
| 124 | return repeated |
| 125 | |
| 126 | |
| 127 | def clean_pages(pages: list[dict[str, Any]], repeated_margin_lines: set[str]) -> list[dict[str, Any]]: |
no test coverage detected