Function collect_papers

scripts/collect_papers_by_date.py:77–134

收集指定日期范围内的论文 Args: start_date: 开始日期 (YYYYMMDD) end_date: 结束日期 (YYYYMMDD) categories: arXiv 类别列表 limit: 最大抓取数量 Returns: 新增论文数量

(
    start_date: str,
    end_date: str,
    categories: List[str] = None,
    limit: int = 500,
)

def collect_papers(
    start_date: str,
    end_date: str,
    categories: List[str] = None,
    limit: int = 500,
) -> int:
    """
    收集指定日期范围内的论文

    Args:
        start_date: 开始日期 (YYYYMMDD)
        end_date: 结束日期 (YYYYMMDD)
        categories: arXiv 类别列表
        limit: 最大抓取数量

    Returns:
        新增论文数量
    """
    conn = init_db()

    # 获取当前论文数量
    before_count = conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0]
    print(f"Before: {before_count} papers in database")

    # 抓取论文
    print(f"\nFetching papers from {start_date} to {end_date}...")
    print(f"Categories: {categories or 'all'}")
    print(f"Limit: {limit}")

    papers = fetch_by_date(
        start_date=start_date,
        end_date=end_date,
        categories=categories,
        limit=limit,
    )

    if papers is None:
        papers = []

    print(f"\nFetched {len(papers)} papers from arXiv")

    # 保存到数据库（去重）
    new_count = 0
    for paper in papers:
        if not paper_exists(conn, paper["arxiv_id"]):
            save_paper(conn, paper)
            new_count += 1
            print(f"  + {paper['arxiv_id']}: {paper['title'][:50]}...")
        else:
            print(f"  ~ {paper['arxiv_id']}: already exists")

    # 统计结果
    after_count = conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0]
    print(f"\nAfter: {after_count} papers in database")
    print(f"New papers added: {new_count}")

    conn.close()
    return new_count