Function _fetch_google_scholar_profile_pages

agents/coldstart-agent/main.py:1263–1323

Fetch one or more Scholar profile pages with lightweight fallbacks.

(scholar_url: str)

def _fetch_google_scholar_profile_pages(scholar_url: str) -> Dict[str, Any]:
    """Fetch one or more Scholar profile pages with lightweight fallbacks."""
    timeout_seconds = max(5, _get_env_int("PAPERFLOW_SCHOLAR_TIMEOUT", SCHOLAR_TIMEOUT_SECONDS))
    publication_limit = max(1, _get_env_int("PAPERFLOW_SCHOLAR_PUBLICATION_LIMIT", SCHOLAR_PUBLICATION_LIMIT))
    page_size = max(20, _get_env_int("PAPERFLOW_SCHOLAR_PAGE_SIZE", 100))
    max_pages = max(1, _get_env_int("PAPERFLOW_SCHOLAR_MAX_PAGES", SCHOLAR_MAX_PAGES))

    session = requests.Session()
    errors: List[str] = []
    profile: Dict[str, Any] = {}

    for page_index in range(max_pages):
        cstart = page_index * page_size
        if len(profile.get("all_publications", [])) >= publication_limit:
            break

        candidate_urls = [
            _build_scholar_profile_url(scholar_url, cstart=cstart, pagesize=page_size, hl="en"),
            _build_scholar_profile_url(scholar_url, cstart=cstart, pagesize=20, hl="en"),
            _build_scholar_profile_url(scholar_url, cstart=cstart, pagesize=20, hl="zh-CN"),
        ]

        page_data: Dict[str, Any] = {}
        page_loaded = False
        for candidate_url in candidate_urls:
            try:
                response = session.get(candidate_url, headers=SCHOLAR_HEADERS, timeout=timeout_seconds)
                response.raise_for_status()
                parsed = _parse_google_scholar_profile_html(response.text)
                if parsed.get("blocked"):
                    errors.append("Scholar returned an anti-bot / captcha page")
                    continue
                if not parsed.get("interests") and not parsed.get("all_publications") and not parsed.get("publications"):
                    fallback_excerpt = _build_scholar_fallback_excerpt(response.text)
                    if fallback_excerpt:
                        parsed["fallback_excerpt"] = fallback_excerpt
                    errors.append("Scholar page loaded but structured profile signals were empty")
                    continue
                page_data = parsed
                page_loaded = True
                break
            except requests.RequestException as exc:
                errors.append(f"{type(exc).__name__}: {exc}")

        if not page_loaded:
            if profile.get("all_publications") or profile.get("interests"):
                break
            raise ValueError("failed to fetch Google Scholar profile; " + "; ".join(errors[-3:]))

        profile = _merge_scholar_page_data(profile, page_data, publication_limit)

        current_page_publications = page_data.get("all_publications") or page_data.get("publications") or []
        if len(current_page_publications) < page_size:
            break

    if not profile.get("interests") and not profile.get("publications"):
        raise ValueError("no scholar interests or publication rows found")