Docip crawler, https://www.docip.net/data/free.json
| 10 | |
| 11 | |
| 12 | class DocipCrawler(BaseCrawler): |
| 13 | """ |
| 14 | Docip crawler, https://www.docip.net/data/free.json |
| 15 | """ |
| 16 | urls = [BASE_URL.format(date=time.strftime("%Y%m%d", time.localtime()))] |
| 17 | |
| 18 | def parse(self, html): |
| 19 | """ |
| 20 | parse html file to get proxies |
| 21 | :return: |
| 22 | """ |
| 23 | try: |
| 24 | result = json.loads(html) |
| 25 | proxy_list = result['data'] |
| 26 | for proxy_item in proxy_list: |
| 27 | host = proxy_item['ip'] |
| 28 | port = host.split(':')[-1] |
| 29 | yield Proxy(host=host, port=port) |
| 30 | except json.JSONDecodeError: |
| 31 | print("json.JSONDecodeError") |
| 32 | return |
| 33 | |
| 34 | |
| 35 | if __name__ == '__main__': |