Function fetch_and_parse

scripts/fetch_eurlex_fulltext.py:470–518

Fetch EUR-Lex HTML and parse to markdown. Retries on HTTP 202 (Accepted) which EUR-Lex returns when generating large documents asynchronously.

(html_url: str)

def fetch_and_parse(html_url: str) -> Optional[tuple[str, str]]:
    """Fetch EUR-Lex HTML and parse to markdown.

    Retries on HTTP 202 (Accepted) which EUR-Lex returns when generating
    large documents asynchronously.
    """
    session = requests.Session()
    session.headers.update(HEADERS)

    for attempt in range(1, MAX_RETRIES + 1):
        try:
            resp = session.get(html_url, timeout=120, allow_redirects=True)
        except Exception as e:
            logger.error(f"  Failed to fetch {html_url}: {e}")
            return None

        if resp.status_code == 202:
            if attempt < MAX_RETRIES:
                logger.info(
                    f"  HTTP 202 Accepted (attempt {attempt}/{MAX_RETRIES}), "
                    f"retrying in {RETRY_DELAY}s...")
                time.sleep(RETRY_DELAY)
                continue
            else:
                logger.warning(
                    f"  HTTP 202 after {MAX_RETRIES} attempts, giving up")
                return None

        if resp.status_code != 200:
            logger.error(f"  HTTP {resp.status_code} for {html_url}")
            return None

        break

    content_type = resp.headers.get("content-type", "").lower()
    if "html" not in content_type:
        logger.error(f"  Unexpected content type: {content_type}")
        return None

    html = resp.text
    logger.info(f"  Downloaded {len(html):,} bytes")

    title, markdown = parse_eurlex_html(html)

    if not markdown or len(markdown) < 100:
        logger.warning(f"  Extracted text too short ({len(markdown)} chars)")
        return None

    return (title, markdown)


# ---------------------------------------------------------------------------