Class FDAFetcher

scripts/fetch_fulltext.py:89–163

Fetch FDA guidance documents using curl-cffi to bypass Akamai bot detection.

    "fda-remanufacturing-2024": "https://www.fda.gov/media/150141/download",
}


class FDAFetcher:
    """Fetch FDA guidance documents using curl-cffi to bypass Akamai bot detection."""

    FDA_GUIDANCE_BASE = "https://www.fda.gov/regulatory-information/search-fda-guidance-documents/"
    FDA_MEDIA_PATTERN = re.compile(r"https://www\.fda\.gov/media/(\d+)/download")
    IMPERSONATE = "chrome124"

    @staticmethod
    def _cffi_get(url: str, timeout: int = 30, **kwargs) -> Optional[object]:
        """Use curl-cffi with Chrome TLS impersonation, bypassing proxy for FDA.
        Akamai bot detection blocks requests from proxy/VPN IPs.
        """
        if cffi_requests is not None:
            saved = {k: os.environ.pop(k, None) for k in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY")}
            try:
                session = cffi_requests.Session(impersonate=FDAFetcher.IMPERSONATE)
                return session.get(url, timeout=timeout, allow_redirects=True, **kwargs)
            finally:
                for k, v in saved.items():
                    if v is not None:
                        os.environ[k] = v
        return requests.get(url, headers=REQUEST_HEADERS, timeout=timeout, allow_redirects=True, **kwargs)

    @staticmethod
    def discover_pdf_url(source_url: str) -> Optional[str]:
        """Try to discover the PDF download URL from an FDA guidance page."""
        try:
            resp = FDAFetcher._cffi_get(source_url, timeout=30)
            resp.raise_for_status()
            content_type = resp.headers.get("content-type", "")
            if "pdf" in content_type.lower():
                return source_url

            text = resp.text
            pdf_links = re.findall(
                r'href=["\']?(https://www\.fda\.gov/media/\d+/download)["\']?',
                text,
            )
            if pdf_links:
                return pdf_links[0]

            media_links = re.findall(
                r'href=["\']?(/media/\d+/download)["\']?',
                text,
            )
            if media_links:
                return f"https://www.fda.gov{media_links[0]}"

        except Exception as e:
            logger.warning(f"  Failed to discover PDF URL from {source_url}: {e}")
        return None

    @staticmethod
    def fetch_pdf(pdf_url: str, source_url: str = "") -> Optional[bytes]:
        """Fetch PDF using curl-cffi with Chrome TLS impersonation."""
        try: