Extract meaningful links from an HTML page.
(self, html: str, base_url: str)
| 2523 | return None |
| 2524 | |
| 2525 | def _parse_links(self, html: str, base_url: str) -> list[dict]: |
| 2526 | """Extract meaningful links from an HTML page.""" |
| 2527 | results = [] |
| 2528 | parsed = urlparse(base_url) |
| 2529 | base_domain = f"{parsed.scheme}://{parsed.netloc}" |
| 2530 | |
| 2531 | try: |
| 2532 | from bs4 import BeautifulSoup |
| 2533 | soup = BeautifulSoup(html, "html.parser") |
| 2534 | for a in soup.find_all("a"): |
| 2535 | title = a.get_text(strip=True) |
| 2536 | href = a.get("href", "") |
| 2537 | if not title or len(title) < 10: |
| 2538 | continue |
| 2539 | if href and not href.startswith("http"): |
| 2540 | href = base_domain + href |
| 2541 | date_el = a.find_parent().find("time") if a.find_parent() else None |
| 2542 | date_str = date_el.get("datetime", "")[:10] if date_el else "" |
| 2543 | results.append({ |
| 2544 | "title": title, "link": href, |
| 2545 | "pub_date": date_str, "description": "", |
| 2546 | }) |
| 2547 | except ImportError: |
| 2548 | link_re = re.compile(r'<a[^>]*href="([^"]*)"[^>]*>(.*?)</a>', re.DOTALL) |
| 2549 | for m in link_re.finditer(html): |
| 2550 | title = re.sub(r"<[^>]+>", "", m.group(2)).strip() |
| 2551 | href = m.group(1) |
| 2552 | if title and len(title) > 10: |
| 2553 | if not href.startswith("http"): |
| 2554 | href = base_domain + href |
| 2555 | results.append({ |
| 2556 | "title": title, "link": href, |
| 2557 | "pub_date": "", "description": "", |
| 2558 | }) |
| 2559 | return results[:30] |
| 2560 | |
| 2561 | |
| 2562 | # --------------------------------------------------------------------------- |