Function fetch_arxiv_papers

deployments/feishu/daily-push-agent/main.py:1205–1242

(fetch_days: int)

            return []
        start_day = (today - timedelta(days=max(1, int(fetch_days or 1)) - 1)).date()
        start_date = start_day.strftime("%Y%m%d")
        per_category_limit = max(1, int(math.ceil(limit_per_source / max(1, len(arxiv_categories)))))

        def fetch_one_category(category: str) -> List[Dict]:
            print(f"Fetching from arXiv {category} ({start_date} to {end_date})...")
            category_papers = arxiv_fetch_by_date(
                start_date=start_date,
                end_date=end_date,
                categories=[category],
                limit=per_category_limit,
            ) or []
            category_papers = [
                paper for paper in category_papers
                if _paper_date_in_window(paper, start_day, end_day)
            ]
            if not category_papers and callable(arxiv_fetch_recent_list_page):
                print(f"  arXiv API returned 0 papers for {category}; fetching recent list page...")
                category_papers = arxiv_fetch_recent_list_page(
                    category=category,
                    limit=per_category_limit,
                ) or []
                category_papers = [
                    paper for paper in category_papers
                    if _paper_date_in_window(paper, start_day, end_day)
                ]
            return category_papers

        papers: List[Dict] = []
        # One independent API call per category; the serial loop this replaces
        # measured 13.1s for 4 categories against ~1.1s for one. executor.map
        # preserves category order so downstream dedup stays deterministic.
        max_workers = min(_env_positive_int("PAPERFLOW_MAX_CONCURRENCY", default=5), len(arxiv_categories))
        if len(arxiv_categories) > 1 and max_workers > 1:
            with ThreadPoolExecutor(max_workers=max_workers) as executor:
                results = list(executor.map(fetch_one_category, arxiv_categories))
        else:
            results = [fetch_one_category(category) for category in arxiv_categories]

        for category_papers in results:
            papers.extend(category_papers)
            if category_papers: