override crawl main method add headers
(self)
| 32 | return |
| 33 | |
| 34 | def crawl(self): |
| 35 | """ |
| 36 | override crawl main method |
| 37 | add headers |
| 38 | """ |
| 39 | headers = { |
| 40 | 'authority': 'proxylist.geonode.com', |
| 41 | 'sec-ch-ua': '" Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"', |
| 42 | 'accept': 'application/json, text/plain, */*', |
| 43 | 'sec-ch-ua-mobile': '?0', |
| 44 | 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.83 Safari/537.36', |
| 45 | 'sec-ch-ua-platform': '"macOS"', |
| 46 | 'origin': 'https://geonode.com', |
| 47 | 'sec-fetch-site': 'same-site', |
| 48 | 'sec-fetch-mode': 'cors', |
| 49 | 'sec-fetch-dest': 'empty', |
| 50 | 'referer': 'https://geonode.com/', |
| 51 | 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,ja;q=0.7', |
| 52 | 'if-none-match': 'W/"c25d-BXjLTmP+/yYXtIz4OEcmdOWSv88"', |
| 53 | } |
| 54 | try: |
| 55 | for url in self.urls: |
| 56 | logger.info(f'fetching {url}') |
| 57 | html = self.fetch(url, headers=headers) |
| 58 | if not html: |
| 59 | continue |
| 60 | time.sleep(.5) |
| 61 | yield from self.process(html, url) |
| 62 | except RetryError: |
| 63 | logger.error( |
| 64 | f'crawler {self} crawled proxy unsuccessfully, ' |
| 65 | 'please check if target url is valid or network issue') |
| 66 | |
| 67 | |
| 68 | if __name__ == '__main__': |
no test coverage detected