Geonode crawler, https://proxylist.geonode.com/
| 10 | |
| 11 | |
| 12 | class GeonodeCrawler(BaseCrawler): |
| 13 | """ |
| 14 | Geonode crawler, https://proxylist.geonode.com/ |
| 15 | """ |
| 16 | urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] |
| 17 | |
| 18 | def parse(self, html): |
| 19 | """ |
| 20 | parse html file to get proxies |
| 21 | :return: |
| 22 | """ |
| 23 | try: |
| 24 | result = json.loads(html) |
| 25 | proxy_list = result['data'] |
| 26 | for proxy_item in proxy_list: |
| 27 | host = proxy_item['ip'] |
| 28 | port = proxy_item['port'] |
| 29 | yield Proxy(host=host, port=port) |
| 30 | except json.JSONDecodeError: |
| 31 | print("json.JSONDecodeError") |
| 32 | return |
| 33 | |
| 34 | def crawl(self): |
| 35 | """ |
| 36 | override crawl main method |
| 37 | add headers |
| 38 | """ |
| 39 | headers = { |
| 40 | 'authority': 'proxylist.geonode.com', |
| 41 | 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"', |
| 42 | 'accept': 'application/json, text/plain, */*', |
| 43 | 'sec-ch-ua-mobile': '?0', |
| 44 | 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', |
| 45 | 'sec-ch-ua-platform': '"macOS"', |
| 46 | 'origin': 'https://geonode.com', |
| 47 | 'sec-fetch-site': 'same-site', |
| 48 | 'sec-fetch-mode': 'cors', |
| 49 | 'sec-fetch-dest': 'empty', |
| 50 | 'referer': 'https://geonode.com/', |
| 51 | 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7', |
| 52 | 'if-none-match': 'W/"c25d-BXjLTmP+/yYXtIz4OEcmdOWSv88"', |
| 53 | } |
| 54 | try: |
| 55 | for url in self.urls: |
| 56 | logger.info(f'fetching {url}') |
| 57 | html = self.fetch(url, headers=headers) |
| 58 | if not html: |
| 59 | continue |
| 60 | time.sleep(.5) |
| 61 | yield from self.process(html, url) |
| 62 | except RetryError: |
| 63 | logger.error( |
| 64 | f'crawler {self} crawled proxy unsuccessfully, ' |
| 65 | 'please check if target url is valid or network issue') |
| 66 | |
| 67 | |
| 68 | if __name__ == '__main__': |