zhandaye crawler, https://www.zdaye.com/dayProxy/
| 10 | |
| 11 | |
| 12 | class ZhandayeCrawler(BaseCrawler): |
| 13 | """ |
| 14 | zhandaye crawler, https://www.zdaye.com/dayProxy/ |
| 15 | """ |
| 16 | urls_catalog = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE)] |
| 17 | headers = { |
| 18 | 'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36' |
| 19 | } |
| 20 | urls = [] |
| 21 | ignore = True |
| 22 | |
| 23 | def crawl(self): |
| 24 | self.crawl_catalog() |
| 25 | yield from super().crawl() |
| 26 | |
| 27 | def crawl_catalog(self): |
| 28 | for url in self.urls_catalog: |
| 29 | logger.info(f'fetching {url}') |
| 30 | html = self.fetch(url, headers=self.headers) |
| 31 | self.parse_catalog(html) |
| 32 | |
| 33 | def parse_catalog(self, html): |
| 34 | """ |
| 35 | parse html file to get proxies |
| 36 | :return: |
| 37 | """ |
| 38 | doc = pq(html) |
| 39 | for item in doc('#J_posts_list .thread_item div div p a').items(): |
| 40 | url = 'https://www.zdaye.com' + item.attr('href') |
| 41 | logger.info(f'get detail url: {url}') |
| 42 | self.urls.append(url) |
| 43 | |
| 44 | def parse(self, html): |
| 45 | doc = pq(html) |
| 46 | trs = doc('.cont br').items() |
| 47 | for tr in trs: |
| 48 | line = tr[0].tail |
| 49 | match = re.search(r'(\d+\.\d+\.\d+\.\d+):(\d+)', line) |
| 50 | if match: |
| 51 | host = match.group(1) |
| 52 | port = match.group(2) |
| 53 | yield Proxy(host=host, port=port) |
| 54 | |
| 55 | |
| 56 | if __name__ == '__main__': |