Function parse_research_homepage

agents/coldstart-agent/main.py:1152–1201

Fetch a personal homepage and derive a homepage-first bootstrap fragment.

(homepage_url: str, use_llm: bool = True)

def parse_research_homepage(homepage_url: str, use_llm: bool = True) -> Dict[str, Any]:
    """Fetch a personal homepage and derive a homepage-first bootstrap fragment."""
    candidate_url = _collapse_whitespace(homepage_url)
    if not candidate_url:
        raise ValueError("empty homepage url")
    if "://" not in candidate_url:
        candidate_url = f"https://{candidate_url.lstrip('/')}"

    response = requests.get(
        candidate_url,
        headers=SCHOLAR_HEADERS,
        timeout=max(5, _get_env_int("PAPERFLOW_HOMEPAGE_TIMEOUT", HOMEPAGE_TIMEOUT_SECONDS)),
    )
    response.raise_for_status()

    homepage_profile = _extract_homepage_sections(response.text)
    homepage_profile["url"] = candidate_url

    explicit_texts = [
        *(homepage_profile.get("sections", {}).get("research_interests", []) or []),
        *(homepage_profile.get("sections", {}).get("research", []) or []),
        *(homepage_profile.get("keywords") or []),
    ]
    secondary_texts = [
        *(homepage_profile.get("sections", {}).get("about", []) or []),
    ]
    publication_titles = [
        *(homepage_profile.get("sections", {}).get("projects", []) or []),
    ]
    if homepage_profile.get("description"):
        secondary_texts.append(str(homepage_profile["description"]))

    parsed_profile = _build_bootstrap_fragment_from_structured_sources(
        explicit_texts=explicit_texts,
        secondary_texts=secondary_texts,
        publication_titles=publication_titles,
        use_llm=use_llm,
    )

    notes: List[str] = []
    if explicit_texts:
        notes.append("已优先根据个人主页研究方向生成冷启动画像。")
    elif secondary_texts:
        notes.append("已结合个人主页摘要信息补充冷启动画像。")

    return {
        "homepage_profile": homepage_profile,
        "parsed_profile": parsed_profile,
        "direction_explanations": notes,
    }


def _merge_scholar_page_data(base: Dict[str, Any], incoming: Dict[str, Any], publication_limit: int) -> Dict[str, Any]: