Process a single entry: discover PDF, download, extract, save.
(entry: dict, dry_run: bool = False)
| 255 | |
| 256 | |
| 257 | def process_entry(entry: dict, dry_run: bool = False) -> bool: |
| 258 | """Process a single entry: discover PDF, download, extract, save.""" |
| 259 | title = entry.get("title", {}) |
| 260 | if isinstance(title, dict): |
| 261 | display = title.get("en", "") or title.get("zh", "") |
| 262 | else: |
| 263 | display = str(title) |
| 264 | eid = entry.get("id", "") |
| 265 | source_url = entry.get("source_url", "") |
| 266 | fw = entry["_framework"] |
| 267 | |
| 268 | logger.info(f"Processing: {eid} - {display[:60]}") |
| 269 | |
| 270 | if not source_url: |
| 271 | logger.warning(f" SKIP: No source_url") |
| 272 | return False |
| 273 | |
| 274 | if fulltext_exists(entry): |
| 275 | logger.info(f" SKIP: Fulltext already exists") |
| 276 | return False |
| 277 | |
| 278 | pdf_url = KNOWN_PDF_URLS.get(eid) |
| 279 | if pdf_url: |
| 280 | logger.info(f" Using known PDF URL") |
| 281 | elif fw == "fda" or "fda.gov" in source_url: |
| 282 | if source_url.endswith("/download"): |
| 283 | pdf_url = source_url |
| 284 | else: |
| 285 | logger.info(f" Discovering PDF URL from FDA page...") |
| 286 | pdf_url = FDAFetcher.discover_pdf_url(source_url) |
| 287 | elif source_url.lower().endswith(".pdf"): |
| 288 | pdf_url = source_url |
| 289 | else: |
| 290 | result = GenericFetcher.fetch(source_url) |
| 291 | if result and result[0] == "pdf": |
| 292 | pdf_url = source_url |
| 293 | |
| 294 | if not pdf_url: |
| 295 | logger.warning(f" SKIP: Could not find PDF URL") |
| 296 | return False |
| 297 | |
| 298 | logger.info(f" PDF URL: {pdf_url}") |
| 299 | |
| 300 | if dry_run: |
| 301 | logger.info(f" DRY RUN: Would download and extract") |
| 302 | return True |
| 303 | |
| 304 | if fitz is None: |
| 305 | logger.error(f" ERROR: PyMuPDF not installed, cannot extract text") |
| 306 | return False |
| 307 | |
| 308 | logger.info(f" Downloading PDF...") |
| 309 | pdf_bytes = FDAFetcher.fetch_pdf(pdf_url, source_url=source_url) |
| 310 | if not pdf_bytes: |
| 311 | return False |
| 312 | |
| 313 | logger.info(f" Extracting text ({len(pdf_bytes) / 1024:.1f} KB)...") |
| 314 | try: |
no test coverage detected