Function parse_or_search_proxy

scrapegraphai/utils/proxy_rotation.py:191–211

Parses a proxy configuration or searches for a matching one via broker.

(proxy: Proxy)

def parse_or_search_proxy(proxy: Proxy) -> ProxySettings:
    """
    Parses a proxy configuration or searches for a matching one via broker.
    """
    assert "server" in proxy, "Missing 'server' field in the proxy configuration."

    parsed_url = urlparse(proxy["server"])
    server_address = parsed_url.hostname

    if server_address is None:
        raise ValueError(f"Invalid proxy server format: {proxy['server']}")

    # Accept both IP addresses and domain names like 'gate.nodemaven.com'
    if is_ipv4_address(server_address) or re.match(
        r"^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", server_address
    ):
        return _parse_proxy(proxy)

    assert proxy["server"] == "broker", f"Unknown proxy server type: {proxy['server']}"

    return _search_proxy(proxy)