Function fetch_and_parse

scripts/fetch_eu_mdr_standards.py:257–327

Step 1: Fetch from EUR-Lex and parse standards list.

(args)

def fetch_and_parse(args) -> list[dict]:
    """Step 1: Fetch from EUR-Lex and parse standards list."""
    print("\n=== Step 1: Fetching consolidated Implementing Decision 2021/1182 ===")

    html = None
    for url in CONSOLIDATED_VERSIONS:
        html = fetch_html(url, "consolidated_2021_1182")
        if html:
            break

    if not html:
        print("ERROR: Could not fetch any consolidated version")
        return []

    standards = parse_annex_table(html)
    print(f"  Parsed {len(standards)} standards from consolidated version")

    # Step 2: Fetch and apply 2026/193 amendment
    print("\n=== Step 2: Checking 2026/193 amendment ===")
    amend_html = fetch_html(AMENDMENT_2026_193_URL, "amendment_2026_193")
    if amend_html:
        new_stds = parse_amendment_additions(amend_html)
        existing_numbers = {s["number"] for s in standards}
        added = 0
        for std_num in new_stds:
            if std_num not in existing_numbers:
                # Check if this is an amendment to an existing standard
                is_amendment = "/A" in std_num
                if is_amendment:
                    # Find base standard and add as amendment
                    base = re.match(r"(EN\s+(?:ISO|IEC)\s+[\d\-]+(?::[\d]+)?)", std_num)
                    if base:
                        base_num = base.group(1)
                        for s in standards:
                            if s["number"] == base_num:
                                if "amendments" not in s:
                                    s["amendments"] = []
                                if std_num not in s["amendments"]:
                                    s["amendments"].append(std_num)
                                break
                else:
                    standards.append({
                        "number": std_num,
                        "title": "",  # Title needs manual review
                        "row_number": str(len(standards) + 1),
                        "source": "2026/193",
                        "needs_review": True,
                    })
                    added += 1
        print(f"  2026/193: {added} new standards added, amendments updated")
    else:
        print("  WARNING: Could not fetch 2026/193 amendment")

    # Classify all standards
    for s in standards:
        s["category"] = classify_standard(s)

    # Save raw parsed data