(self, source_id: str, source: dict)
| 1189 | self.seed_mode = seed_mode |
| 1190 | |
| 1191 | def check(self, source_id: str, source: dict) -> Optional[dict]: |
| 1192 | url = source["url"] |
| 1193 | try: |
| 1194 | from bs4 import BeautifulSoup |
| 1195 | resp = self.session.get(url, timeout=30) |
| 1196 | resp.raise_for_status() |
| 1197 | resp.encoding = resp.apparent_encoding or "utf-8" |
| 1198 | soup = BeautifulSoup(resp.text, "html.parser") |
| 1199 | |
| 1200 | prev = self.state.get(source_id, {}) |
| 1201 | prev_titles = set(prev.get("seen_titles", [])) |
| 1202 | new_items = [] |
| 1203 | all_titles = list(prev_titles) |
| 1204 | |
| 1205 | for a_tag in soup.find_all("a", href=True): |
| 1206 | text = a_tag.get_text(strip=True)[:200] |
| 1207 | if not text or len(text) < 20: |
| 1208 | continue |
| 1209 | href = a_tag["href"] |
| 1210 | if not href.startswith("http"): |
| 1211 | href = "https://www.argentina.gob.ar" + href |
| 1212 | |
| 1213 | if not self.DEVICE_KW.search(text): |
| 1214 | continue |
| 1215 | |
| 1216 | if text in prev_titles: |
| 1217 | continue |
| 1218 | all_titles.append(text) |
| 1219 | |
| 1220 | dm = re.search( |
| 1221 | r"(\d{1,2})\s+de\s+(\w+)\s+de\s+(\d{4})", text) |
| 1222 | pub_date = "" |
| 1223 | if dm: |
| 1224 | month = self.MONTH_MAP_ES.get(dm.group(2).lower(), "") |
| 1225 | if month: |
| 1226 | pub_date = f"{dm.group(3)}-{month}-{dm.group(1).zfill(2)}" |
| 1227 | |
| 1228 | parent = a_tag.find_parent() |
| 1229 | if parent and not pub_date: |
| 1230 | sibling_text = "" |
| 1231 | prev_sib = parent.find_previous_sibling() |
| 1232 | if prev_sib: |
| 1233 | sibling_text = prev_sib.get_text(strip=True) |
| 1234 | dm2 = re.search( |
| 1235 | r"(\d{1,2})\s+de\s+(\w+)\s+de\s+(\d{4})", sibling_text) |
| 1236 | if dm2: |
| 1237 | month = self.MONTH_MAP_ES.get(dm2.group(2).lower(), "") |
| 1238 | if month: |
| 1239 | pub_date = f"{dm2.group(3)}-{month}-{dm2.group(1).zfill(2)}" |
| 1240 | |
| 1241 | new_items.append({ |
| 1242 | "title": text, |
| 1243 | "link": href, |
| 1244 | "pub_date": pub_date, |
| 1245 | "description": f"ANMAT Alert: {text[:200]}", |
| 1246 | }) |
| 1247 | |
| 1248 | self.state[source_id] = { |
nothing calls this directly
no test coverage detected