Sets up a proxy using ScraperAPI The optional parameters are only for Business and Enterprise plans with ScraperAPI. For more details, https://www.scraperapi.com/documentation/ :Example:: >>> pg = ProxyGenerator() >>> success = pg.ScraperAPI
(self, API_KEY, country_code=None, premium=False, render=False)
| 571 | return True |
| 572 | |
| 573 | def ScraperAPI(self, API_KEY, country_code=None, premium=False, render=False): |
| 574 | """ |
| 575 | Sets up a proxy using ScraperAPI |
| 576 | |
| 577 | The optional parameters are only for Business and Enterprise plans with |
| 578 | ScraperAPI. For more details, https://www.scraperapi.com/documentation/ |
| 579 | |
| 580 | :Example:: |
| 581 | >>> pg = ProxyGenerator() |
| 582 | >>> success = pg.ScraperAPI(API_KEY) |
| 583 | |
| 584 | :param API_KEY: ScraperAPI API Key value. |
| 585 | :type API_KEY: string |
| 586 | :type country_code: string, optional by default None |
| 587 | :type premium: bool, optional by default False |
| 588 | :type render: bool, optional by default False |
| 589 | :returns: whether or not the proxy was set up successfully |
| 590 | :rtype: {bool} |
| 591 | """ |
| 592 | if API_KEY is None: |
| 593 | raise ValueError("ScraperAPI API Key is required.") |
| 594 | |
| 595 | # Get basic account information. This will NOT be counted towards successful API requests. |
| 596 | r = requests.get("http://api.scraperapi.com/account", params={'api_key': API_KEY}).json() |
| 597 | if "error" in r: |
| 598 | self.logger.warning(r["error"]) |
| 599 | return False |
| 600 | |
| 601 | self._API_KEY = API_KEY |
| 602 | self.proxy_mode = ProxyMode.SCRAPERAPI |
| 603 | |
| 604 | r["requestLimit"] = int(r["requestLimit"]) |
| 605 | self.logger.info("Successful ScraperAPI requests %d / %d", |
| 606 | r["requestCount"], r["requestLimit"]) |
| 607 | |
| 608 | # ScraperAPI documentation recommends setting the timeout to 60 seconds |
| 609 | # so it has had a chance to try out all the retries. |
| 610 | # https://www.scraperapi.com/documentation/ |
| 611 | self._TIMEOUT = 60 |
| 612 | |
| 613 | prefix = "http://scraperapi.retry_404=true" |
| 614 | if country_code is not None: |
| 615 | prefix += ".country_code=" + country_code |
| 616 | if premium: |
| 617 | prefix += ".premium=true" |
| 618 | if render: |
| 619 | prefix += ".render=true" |
| 620 | |
| 621 | # Suppress the unavoidable insecure request warnings with ScraperAPI |
| 622 | urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) |
| 623 | |
| 624 | for _ in range(3): |
| 625 | proxy_works = self._use_proxy(http=f'{prefix}:{API_KEY}@proxy-server.scraperapi.com:8001') |
| 626 | if proxy_works: |
| 627 | proxies = {'http://': f"{prefix}:{API_KEY}@proxy-server.scraperapi.com:8001",} |
| 628 | self.logger.info("ScraperAPI proxy setup successfully") |
| 629 | self._new_session(verify=False, proxies=proxies) |
| 630 | return proxy_works |