Fetch HTML from URL with caching.
(url: str, cache_name: str)
| 122 | |
| 123 | |
| 124 | def fetch_html(url: str, cache_name: str) -> Optional[str]: |
| 125 | """Fetch HTML from URL with caching.""" |
| 126 | CACHE_DIR.mkdir(parents=True, exist_ok=True) |
| 127 | cache_file = CACHE_DIR / f"{cache_name}.html" |
| 128 | |
| 129 | # Use cache if less than 24 hours old |
| 130 | if cache_file.exists(): |
| 131 | age_hours = (datetime.now().timestamp() - cache_file.stat().st_mtime) / 3600 |
| 132 | if age_hours < 24: |
| 133 | print(f" Using cached: {cache_file.name} ({age_hours:.1f}h old)") |
| 134 | return cache_file.read_text(encoding="utf-8") |
| 135 | |
| 136 | print(f" Fetching: {url}") |
| 137 | try: |
| 138 | resp = requests.get(url, timeout=30, headers={ |
| 139 | "User-Agent": "Mozilla/5.0 (docmcp-knowledge-bot/2.0)", |
| 140 | "Accept": "text/html", |
| 141 | }) |
| 142 | resp.raise_for_status() |
| 143 | html = resp.text |
| 144 | cache_file.write_text(html, encoding="utf-8") |
| 145 | print(f" Saved to cache: {cache_file.name} ({len(html)} bytes)") |
| 146 | return html |
| 147 | except requests.RequestException as e: |
| 148 | print(f" ERROR fetching {url}: {e}") |
| 149 | return None |
| 150 | |
| 151 | |
| 152 | def parse_annex_table(html: str) -> list[dict]: |