Fetch HTML version from ar5iv (renders math as readable text).
(arxiv_id: str)
| 186 | |
| 187 | |
| 188 | def fetch_ar5iv_html(arxiv_id: str) -> str | None: |
| 189 | """Fetch HTML version from ar5iv (renders math as readable text).""" |
| 190 | base_id = re.sub(r"v\d+$", "", arxiv_id) |
| 191 | html_url = f"https://ar5iv.labs.arxiv.org/html/{base_id}" |
| 192 | print(f"Fetching HTML from {html_url}...") |
| 193 | |
| 194 | try: |
| 195 | resp = requests.get(html_url, timeout=60) |
| 196 | resp.raise_for_status() |
| 197 | |
| 198 | # Basic HTML to text conversion — strip tags but keep structure |
| 199 | text = resp.text |
| 200 | |
| 201 | # Remove script and style blocks |
| 202 | text = re.sub(r"<script[^>]*>.*?</script>", "", text, flags=re.DOTALL) |
| 203 | text = re.sub(r"<style[^>]*>.*?</style>", "", text, flags=re.DOTALL) |
| 204 | |
| 205 | # Convert headers to markdown |
| 206 | for level in range(1, 7): |
| 207 | text = re.sub( |
| 208 | rf"<h{level}[^>]*>(.*?)</h{level}>", |
| 209 | lambda m, lv=level: f"\n{'#' * lv} {m.group(1).strip()}\n", |
| 210 | text, |
| 211 | flags=re.DOTALL, |
| 212 | ) |
| 213 | |
| 214 | # Convert paragraphs to double newlines |
| 215 | text = re.sub(r"<p[^>]*>", "\n\n", text) |
| 216 | text = re.sub(r"</p>", "", text) |
| 217 | |
| 218 | # Convert list items |
| 219 | text = re.sub(r"<li[^>]*>", "\n- ", text) |
| 220 | |
| 221 | # Preserve math elements (ar5iv uses MathML or LaTeX in alt text) |
| 222 | text = re.sub(r'<math[^>]*alttext="([^"]*)"[^>]*>.*?</math>', r"$\1$", text, flags=re.DOTALL) |
| 223 | |
| 224 | # Strip remaining HTML tags |
| 225 | text = re.sub(r"<[^>]+>", "", text) |
| 226 | |
| 227 | # Clean up whitespace |
| 228 | text = re.sub(r"\n{3,}", "\n\n", text) |
| 229 | text = text.strip() |
| 230 | |
| 231 | if len(text) > 500: |
| 232 | print(f" Extracted: {len(text)} characters from HTML") |
| 233 | return text |
| 234 | |
| 235 | print(" WARNING: ar5iv HTML produced insufficient text.", file=sys.stderr) |
| 236 | return None |
| 237 | |
| 238 | except requests.RequestException as e: |
| 239 | print(f" ar5iv fetch failed: {e}", file=sys.stderr) |
| 240 | return None |
| 241 | |
| 242 | |
| 243 | def check_text_quality(text: str) -> bool: |