(self, source_id: str, source: dict)
| 1083 | self.seed_mode = seed_mode |
| 1084 | |
| 1085 | def check(self, source_id: str, source: dict) -> Optional[dict]: |
| 1086 | url = source["url"] |
| 1087 | try: |
| 1088 | from bs4 import BeautifulSoup |
| 1089 | resp = self.session.get(url, timeout=15) |
| 1090 | resp.raise_for_status() |
| 1091 | resp.encoding = resp.apparent_encoding or "utf-8" |
| 1092 | soup = BeautifulSoup(resp.text, "html.parser") |
| 1093 | |
| 1094 | prev = self.state.get(source_id, {}) |
| 1095 | prev_titles = set(prev.get("seen_titles", [])) |
| 1096 | new_items = [] |
| 1097 | all_titles = list(prev_titles) |
| 1098 | |
| 1099 | for article in soup.select("article, .article, .views-row, div.row"): |
| 1100 | a_tag = article.find("a") |
| 1101 | if not a_tag: |
| 1102 | continue |
| 1103 | title = a_tag.get_text(strip=True)[:200] |
| 1104 | if not title or len(title) < 10: |
| 1105 | continue |
| 1106 | href = a_tag.get("href", "") |
| 1107 | if href and not href.startswith("http"): |
| 1108 | href = "https://www.gob.mx" + href |
| 1109 | |
| 1110 | if title in prev_titles: |
| 1111 | continue |
| 1112 | all_titles.append(title) |
| 1113 | article_text = article.get_text(" ", strip=True) |
| 1114 | if not self.DEVICE_KEYWORDS.search(article_text): |
| 1115 | continue |
| 1116 | new_items.append({ |
| 1117 | "title": title, |
| 1118 | "link": href or url, |
| 1119 | "description": f"COFEPRIS Alert: {title[:200]}", |
| 1120 | }) |
| 1121 | |
| 1122 | if not new_items: |
| 1123 | links = soup.find_all("a", href=True) |
| 1124 | for a_tag in links: |
| 1125 | title = a_tag.get_text(strip=True)[:200] |
| 1126 | if not title or len(title) < 15: |
| 1127 | continue |
| 1128 | href = a_tag["href"] |
| 1129 | if "/alertas" not in href and "/retiro" not in href: |
| 1130 | continue |
| 1131 | if not href.startswith("http"): |
| 1132 | href = "https://www.gob.mx" + href |
| 1133 | if title in prev_titles: |
| 1134 | continue |
| 1135 | all_titles.append(title) |
| 1136 | if not self.DEVICE_KEYWORDS.search(title): |
| 1137 | continue |
| 1138 | new_items.append({ |
| 1139 | "title": title, |
| 1140 | "link": href, |
| 1141 | "description": f"COFEPRIS Alert: {title[:200]}", |
| 1142 | }) |
nothing calls this directly
no test coverage detected