Function main

skills/paper2code/scripts/fetch_paper.py:339–432

()

def main():
    if len(sys.argv) < 3:
        print(f"Usage: {sys.argv[0]} <arxiv_id_or_url> <output_dir>", file=sys.stderr)
        sys.exit(1)

    raw_input = sys.argv[1]
    output_dir = Path(sys.argv[2])
    output_dir.mkdir(parents=True, exist_ok=True)

    # Step 1: Normalize ID
    arxiv_id = normalize_arxiv_id(raw_input)
    print(f"Arxiv ID: {arxiv_id}")

    # Step 2: Fetch metadata
    print("\n--- Fetching metadata ---")
    metadata = fetch_metadata(arxiv_id)
    metadata_path = output_dir / "paper_metadata.json"
    with open(metadata_path, "w", encoding="utf-8") as f:
        json.dump(metadata, f, indent=2, ensure_ascii=False)
    print(f"  Title: {metadata['title']}")
    print(f"  Authors: {', '.join(metadata['authors'][:5])}{'...' if len(metadata['authors']) > 5 else ''}")
    print(f"  Categories: {', '.join(metadata['categories'])}")

    # Step 3: Download and extract PDF
    paper_text = None
    pdf_path = output_dir / "paper.pdf"

    print("\n--- Downloading PDF ---")
    if download_pdf(arxiv_id, pdf_path):
        # Try pymupdf4llm first
        print("\n--- Extracting text ---")
        paper_text = extract_with_pymupdf4llm(pdf_path)

        # Check quality
        if paper_text and not check_text_quality(paper_text):
            print("  pymupdf4llm text quality poor, trying pdfplumber...")
            paper_text = None

        # Fallback to pdfplumber
        if paper_text is None:
            paper_text = extract_with_pdfplumber(pdf_path)

        if paper_text and not check_text_quality(paper_text):
            print("  pdfplumber text quality poor, trying ar5iv HTML...")
            paper_text = None

    # Step 4: Fallback to ar5iv HTML
    if paper_text is None:
        print("\n--- Trying ar5iv HTML fallback ---")
        paper_text = fetch_ar5iv_html(arxiv_id)

    # Step 5: Save results
    if paper_text is None:
        print("\nERROR: All extraction methods failed.", file=sys.stderr)
        print("Please download the paper manually and provide the text.", file=sys.stderr)
        sys.exit(1)

    text_path = output_dir / "paper_text.md"