Function precollect_by_day

scripts/precollect_papers_by_day.py:53–101

(
    *,
    start_date: str,
    end_date: str,
    sources: Optional[List[str]],
    limit_per_source: Optional[int],
)

def precollect_by_day(
    *,
    start_date: str,
    end_date: str,
    sources: Optional[List[str]],
    limit_per_source: Optional[int],
) -> None:
    start = _parse_day(start_date)
    end = _parse_day(end_date)
    if end < start:
        raise ValueError("--end-date must be >= --start-date")

    print(f"Pre-collecting papers from {start.date()} to {end.date()}")
    print(f"Sources: {sources or ['arxiv', 'openreview', 'journal']}")
    print(f"Limit per source: {limit_per_source if limit_per_source is not None else 'collector default'}")
    print(f"Initial DB papers: {_count_total_papers()}")

    current = start
    days = 0
    total_reported_new = 0
    while current <= end:
        day_key = current.strftime("%Y%m%d")
        day_label = current.strftime("%Y-%m-%d")
        before_day_count = _count_papers_for_day(current)
        print("\n" + "=" * 72)
        print(f"[{day_label}] collecting one-day paper cache")
        print(f"[{day_label}] existing papers for this date: {before_day_count}")

        reported_new = collect_all_papers(
            start_date=day_key,
            end_date=day_key,
            sources=sources,
            limit_per_source=limit_per_source,
        )

        after_day_count = _count_papers_for_day(current)
        print(
            f"[{day_label}] collector reported new={reported_new}; "
            f"date_count {before_day_count} -> {after_day_count}"
        )

        total_reported_new += int(reported_new or 0)
        days += 1
        current += timedelta(days=1)

    print("\n" + "=" * 72)
    print(f"Pre-collection complete: {days} days")
    print(f"Collector reported total new papers: {total_reported_new}")
    print(f"Final DB papers: {_count_total_papers()}")


def main() -> None: