从单个来源收集论文
(
conn: sqlite3.Connection,
source: str,
start_date: str,
end_date: str,
limit: Optional[int],
extra_params: Optional[Dict] = None,
)
| 110 | |
| 111 | |
| 112 | def collect_from_source( |
| 113 | conn: sqlite3.Connection, |
| 114 | source: str, |
| 115 | start_date: str, |
| 116 | end_date: str, |
| 117 | limit: Optional[int], |
| 118 | extra_params: Optional[Dict] = None, |
| 119 | ) -> int: |
| 120 | """从单个来源收集论文""" |
| 121 | # 不限制数量，使用各 fetcher 的默认行为 |
| 122 | params = {"start_date": start_date, "end_date": end_date} |
| 123 | |
| 124 | # 只在用户明确指定 limit 时才传递 |
| 125 | if limit is not None and limit > 0: |
| 126 | params["limit"] = limit |
| 127 | # 否则让 fetcher 使用自己的默认值（通常是不限制或 fetcher 内部默认值） |
| 128 | |
| 129 | if extra_params: |
| 130 | params.update(extra_params) |
| 131 | |
| 132 | print(f"\n Fetching from {source}...") |
| 133 | |
| 134 | try: |
| 135 | if source == "arxiv": |
| 136 | # arXiv 不限制类别，设置较大的上限（arXiv API 单次最多 1000 篇） |
| 137 | if "limit" not in params: |
| 138 | params["limit"] = 1000 |
| 139 | papers = fetch_arxiv(categories=ARXIV_CATEGORIES, **params) |
| 140 | elif source == "openreview": |
| 141 | # OpenReview 不限制数量 |
| 142 | if "limit" not in params: |
| 143 | params["limit"] = 1000 |
| 144 | papers = fetch_openreview(conferences=["iclr", "neurips", "icml", "acl", "emnlp", "cvpr"], **params) |
| 145 | elif source == "journal": |
| 146 | # Journals 不限制数量 |
| 147 | if "limit" not in params: |
| 148 | params["limit"] = 500 |
| 149 | papers = fetch_journal(journals=JOURNALS, **params) |
| 150 | else: |
| 151 | print(f" Unknown source: {source}") |
| 152 | return 0 |
| 153 | |
| 154 | if papers is None: |
| 155 | papers = [] |
| 156 | |
| 157 | print(f" Fetched {len(papers)} papers") |
| 158 | |
| 159 | new_count = 0 |
| 160 | for paper in papers: |
| 161 | normalized = normalize_paper(paper, source) |
| 162 | paper_id = normalized.get("arxiv_id") or normalized.get("doi") or normalized.get("title") |
| 163 | |
| 164 | if not paper_exists(conn, paper_id, source): |
| 165 | save_paper(conn, normalized) |
| 166 | new_count += 1 |
| 167 | print(f" + {paper_id}: {normalized['title'][:50]}...") |
| 168 | else: |
| 169 | print(f" ~ {paper_id}: already exists") |
no test coverage detected