| 262 | fields = "title,abstract,authors,year,url,venue,externalIds,publicationDate" |
| 263 | |
| 264 | def fetch_one_query(query: str) -> List[Dict]: |
| 265 | params = urllib.parse.urlencode({"query": query, "limit": per_query_limit, "fields": fields}) |
| 266 | url = f"https://api.semanticscholar.org/graph/v1/paper/search?{params}" |
| 267 | try: |
| 268 | request = urllib.request.Request( |
| 269 | url, |
| 270 | headers={"User-Agent": "PaperFlow Desktop", "x-api-key": api_key}, |
| 271 | ) |
| 272 | with urllib.request.urlopen(request, timeout=20) as response: |
| 273 | payload = json.loads(response.read().decode("utf-8")) |
| 274 | except Exception as exc: |
| 275 | print(f" Semantic Scholar fetch error for {query}: {exc}") |
| 276 | return [] |
| 277 | |
| 278 | query_papers: List[Dict] = [] |
| 279 | for item in payload.get("data", []) or []: |
| 280 | title = str(item.get("title") or "").strip() |
| 281 | if not title: |
| 282 | continue |
| 283 | published = str(item.get("publicationDate") or "").strip() |
| 284 | if published: |
| 285 | try: |
| 286 | published_date = datetime.strptime(published[:10], "%Y-%m-%d").date() |
| 287 | if published_date < min_date or published_date > end_date: |
| 288 | continue |
| 289 | except ValueError: |
| 290 | pass |
| 291 | external_ids = item.get("externalIds") or {} |
| 292 | authors = [author.get("name", "") for author in item.get("authors", []) or [] if author.get("name")] |
| 293 | query_papers.append( |
| 294 | { |
| 295 | "title": title, |
| 296 | "authors": authors, |
| 297 | "abstract": item.get("abstract") or "", |
| 298 | "arxiv_id": external_ids.get("ArXiv", ""), |
| 299 | "doi": external_ids.get("DOI", ""), |
| 300 | "url": item.get("url") or "", |
| 301 | "paper_url": item.get("url") or "", |
| 302 | "source": "semantic_scholar", |
| 303 | "venue": item.get("venue") or "Semantic Scholar", |
| 304 | "publish_date": published or str(item.get("year") or ""), |
| 305 | "categories": ["semantic_scholar", query], |
| 306 | "metadata": {"semantic_scholar_query": query, "semantic_scholar_ids": external_ids}, |
| 307 | } |
| 308 | ) |
| 309 | return query_papers |
| 310 | |
| 311 | papers: List[Dict] = [] |
| 312 | max_workers = min(_env_positive_int("PAPERFLOW_MAX_CONCURRENCY", default=5), len(queries)) |