Function scrape_do_fetch

scrapegraphai/docloaders/scrape_do.py:14–50

Fetches the IP address of the machine associated with the given URL using Scrape.do. Args: token (str): The API token for Scrape.do service. target_url (str): A valid web page URL to fetch its associated IP address. use_proxy (bool): Whether to use Scrape.do proxy m

(
    token, target_url, use_proxy=False, geoCode=None, super_proxy=False
)

def scrape_do_fetch(
    token, target_url, use_proxy=False, geoCode=None, super_proxy=False
):
    """
    Fetches the IP address of the machine associated with the given URL using Scrape.do.

    Args:
        token (str): The API token for Scrape.do service.
        target_url (str): A valid web page URL to fetch its associated IP address.
        use_proxy (bool): Whether to use Scrape.do proxy mode. Default is False.
        geoCode (str, optional): Specify the country code for
        geolocation-based proxies. Default is None.
        super_proxy (bool): If True, use Residential & Mobile Proxy Networks. Default is False.

    Returns:
        str: The raw response from the target URL.
    """
    encoded_url = urllib.parse.quote(target_url)
    if use_proxy:
        proxy_scrape_do_url = os.getenv("PROXY_SCRAPE_DO_URL", "proxy.scrape.do:8080")
        proxy_mode_url = f"http://{token}:@{proxy_scrape_do_url}"
        proxies = {
            "http": proxy_mode_url,
            "https": proxy_mode_url,
        }
        params = (
            {"geoCode": geoCode, "super": str(super_proxy).lower()} if geoCode else {}
        )
        response = requests.get(
            target_url, proxies=proxies, verify=False, params=params
        )
    else:
        api_scrape_do_url = os.getenv("API_SCRAPE_DO_URL", "api.scrape.do")
        url = f"http://{api_scrape_do_url}?token={token}&url={encoded_url}"
        response = requests.get(url)

    return response.text