Parse MFDS English board page (supports m_40, m_41, m_61).
(html: str, base_url: str)
| 2396 | |
| 2397 | @staticmethod |
| 2398 | def _parse_news(html: str, base_url: str) -> list[dict]: |
| 2399 | """Parse MFDS English board page (supports m_40, m_41, m_61).""" |
| 2400 | results = [] |
| 2401 | try: |
| 2402 | from bs4 import BeautifulSoup |
| 2403 | soup = BeautifulSoup(html, "html.parser") |
| 2404 | for a in soup.find_all("a", href=re.compile(r"view\.do")): |
| 2405 | title = a.get_text(strip=True) |
| 2406 | href = a.get("href", "") |
| 2407 | if not title or len(title) < 10: |
| 2408 | continue |
| 2409 | href = href.replace("&", "&") |
| 2410 | if href.startswith("./"): |
| 2411 | href = base_url + href[2:] |
| 2412 | elif href and not href.startswith("http"): |
| 2413 | href = base_url + href |
| 2414 | date_str = "" |
| 2415 | date_in_title = re.search( |
| 2416 | r"(\d{4})[.\s]+(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+(\d{1,2})", |
| 2417 | title, |
| 2418 | ) |
| 2419 | if date_in_title: |
| 2420 | pass |
| 2421 | row = a.find_parent("tr") or a.find_parent("li") |
| 2422 | if row: |
| 2423 | for el in row.find_all(["td", "span", "p"]): |
| 2424 | dm = re.search(r"(\d{4})[.\-/](\d{2})[.\-/](\d{2})", el.get_text()) |
| 2425 | if dm: |
| 2426 | date_str = f"{dm.group(1)}-{dm.group(2)}-{dm.group(3)}" |
| 2427 | break |
| 2428 | if not date_str: |
| 2429 | dm_raw = re.search(r"(\d{4})-(\d{2})-(\d{2})", html[html.find(title[:20]):html.find(title[:20])+500] if title[:20] in html else "") |
| 2430 | if dm_raw: |
| 2431 | date_str = f"{dm_raw.group(1)}-{dm_raw.group(2)}-{dm_raw.group(3)}" |
| 2432 | if href.endswith("down.do") or "/down.do?" in href: |
| 2433 | continue |
| 2434 | results.append({ |
| 2435 | "title": title, "link": href, |
| 2436 | "pub_date": date_str, "description": "", |
| 2437 | }) |
| 2438 | except ImportError: |
| 2439 | link_re = re.compile(r'<a[^>]*href="(\./view\.do[^"]*)"[^>]*>(.*?)</a>', re.DOTALL) |
| 2440 | for m in link_re.finditer(html): |
| 2441 | title = re.sub(r"<[^>]+>", "", m.group(2)).strip() |
| 2442 | href = m.group(1).replace("&", "&") |
| 2443 | if title and len(title) > 10: |
| 2444 | href = base_url + href[2:] |
| 2445 | results.append({ |
| 2446 | "title": title, "link": href, |
| 2447 | "pub_date": "", "description": "", |
| 2448 | }) |
| 2449 | seen = set() |
| 2450 | deduped = [] |
| 2451 | for r in results: |
| 2452 | key = r["title"][:50] |
| 2453 | if key not in seen: |
| 2454 | seen.add(key) |
| 2455 | deduped.append(r) |