Function process_entry

scripts/fetch_imdrf_fulltext.py:157–217

Process a single IMDRF index entry: download DOCX and convert to Markdown.

(entry: dict)

def process_entry(entry: dict) -> bool:
    """Process a single IMDRF index entry: download DOCX and convert to Markdown."""
    doc_id = entry["id"]
    title = entry["title"]
    title_en = title.get("en", "") if isinstance(title, dict) else str(title)
    source_url = entry.get("source_url", "")
    doc_number = entry.get("document_number", "")

    output_path = OUTPUT_DIR / f"{doc_id}.md"
    if output_path.exists() and not FORCE_RECONVERT:
        size = output_path.stat().st_size
        if size > 500:
            print(f"  [SKIP] {doc_id} -- already exists ({size} bytes)")
            return False

    print(f"  Processing: {doc_id}")
    print(f"    Title: {title_en}")
    print(f"    URL: {source_url}")

    if not source_url:
        print(f"    [WARN] No source URL")
        return False

    docx_url, pdf_url = fetch_docx_url(source_url)
    print(f"    DOCX: {docx_url or '(none)'}")
    print(f"    PDF:  {pdf_url or '(none)'}")

    if not docx_url:
        print(f"    [WARN] No DOCX link found, skipping")
        return False

    try:
        print(f"    Downloading DOCX...")
        resp = requests.get(docx_url, headers=HEADERS, timeout=60)
        resp.raise_for_status()
    except Exception as e:
        print(f"    [ERROR] Download failed: {e}")
        return False

    with tempfile.NamedTemporaryFile(suffix=".docx", delete=False) as tmp:
        tmp.write(resp.content)
        tmp_path = tmp.name

    try:
        print(f"    Converting to Markdown...")
        md = docx_to_markdown(tmp_path)

        header = f"# {title_en}\n\n"
        if doc_number:
            header += f"**Document Number**: {doc_number}\n\n"
        header += f"**Source**: [{source_url}]({source_url})\n\n---\n\n"

        full_md = header + md
        output_path.write_text(full_md, encoding="utf-8")
        print(f"    [OK] Written {len(full_md)} chars -> {output_path.name}")
        return True
    except Exception as e:
        print(f"    [ERROR] Conversion failed: {e}")