Function collect_all_papers

scripts/collect_all_papers.py:178–221

从所有来源收集论文 Args: start_date: 开始日期 (YYYYMMDD) end_date: 结束日期 (YYYYMMDD) sources: 来源列表 limit_per_source: 每个来源的抓取数量限制 Returns: 新增论文数量

(
    start_date: str,
    end_date: str,
    sources: List[str] = None,
    limit_per_source: int = 200,
)

def collect_all_papers(
    start_date: str,
    end_date: str,
    sources: List[str] = None,
    limit_per_source: int = 200,
) -> int:
    """
    从所有来源收集论文

    Args:
        start_date: 开始日期 (YYYYMMDD)
        end_date: 结束日期 (YYYYMMDD)
        sources: 来源列表
        limit_per_source: 每个来源的抓取数量限制

    Returns:
        新增论文数量
    """
    conn = init_db()

    before_count = conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0]
    print(f"Before: {before_count} papers in database")

    if sources is None:
        sources = ["arxiv", "openreview", "journal"]

    total_new = 0
    for source in sources:
        new_count = collect_from_source(
            conn=conn,
            source=source,
            start_date=start_date,
            end_date=end_date,
            limit=limit_per_source,
        )
        total_new += new_count
        print(f"  [{source}] Added {new_count} new papers")

    after_count = conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0]
    print(f"\nAfter: {after_count} papers in database")
    print(f"Total new papers added: {total_new}")

    conn.close()
    return total_new


def main():