Fetch EC page, parse all MDCG refs, compare with DB.
(self, source_id: str, source: dict)
| 1082 | self.db_comparator = db_comparator |
| 1083 | |
| 1084 | def check(self, source_id: str, source: dict) -> Optional[dict]: |
| 1085 | """Fetch EC page, parse all MDCG refs, compare with DB.""" |
| 1086 | try: |
| 1087 | resp = self.session.get(self.MDCG_URL, timeout=30) |
| 1088 | resp.raise_for_status() |
| 1089 | except Exception as e: |
| 1090 | print(f" ERROR fetching EC MDCG page: {e}") |
| 1091 | return None |
| 1092 | |
| 1093 | page_text = resp.text |
| 1094 | official_docs = self._parse_mdcg_refs(page_text) |
| 1095 | if not official_docs: |
| 1096 | print(f" WARNING: No MDCG refs parsed from EC page") |
| 1097 | return None |
| 1098 | |
| 1099 | print(f" INFO: Parsed {len(official_docs)} unique MDCG documents from EC page") |
| 1100 | |
| 1101 | new_items = [] |
| 1102 | update_items = [] |
| 1103 | |
| 1104 | for doc_id, info in official_docs.items(): |
| 1105 | classification, desc = self.db_comparator.classify( |
| 1106 | "eu_mdr/mdcg", info["title"], info.get("url", ""), info.get("date", "") |
| 1107 | ) |
| 1108 | if classification == "new": |
| 1109 | new_items.append({ |
| 1110 | "title": info["title"], |
| 1111 | "link": info.get("url", ""), |
| 1112 | "pub_date": info.get("date", ""), |
| 1113 | "doc_id": doc_id, |
| 1114 | "revision": info.get("revision", ""), |
| 1115 | "db_classification": "new", |
| 1116 | "db_description": desc, |
| 1117 | }) |
| 1118 | elif classification == "update": |
| 1119 | update_items.append({ |
| 1120 | "title": info["title"], |
| 1121 | "link": info.get("url", ""), |
| 1122 | "pub_date": info.get("date", ""), |
| 1123 | "doc_id": doc_id, |
| 1124 | "revision": info.get("revision", ""), |
| 1125 | "db_classification": "update", |
| 1126 | "db_description": desc, |
| 1127 | }) |
| 1128 | |
| 1129 | self.state[source_id] = { |
| 1130 | "url": source["url"], |
| 1131 | "last_checked": datetime.now().isoformat(), |
| 1132 | "official_count": len(official_docs), |
| 1133 | } |
| 1134 | |
| 1135 | all_detected = new_items + update_items |
| 1136 | if not all_detected: |
| 1137 | return None |
| 1138 | |
| 1139 | result = _make_update( |
| 1140 | source_id, source, "ec_page_scrape", |
| 1141 | f"{len(new_items)} new + {len(update_items)} updated MDCG document(s) " |
nothing calls this directly
no test coverage detected