parse html file to get proxies :return:
(self, html)
| 16 | urls = [BASE_URL.format(page=page) for page in range(1, MAX_PAGE + 1)] |
| 17 | |
| 18 | def parse(self, html): |
| 19 | """ |
| 20 | parse html file to get proxies |
| 21 | :return: |
| 22 | """ |
| 23 | try: |
| 24 | result = json.loads(html) |
| 25 | proxy_list = result['data'] |
| 26 | for proxy_item in proxy_list: |
| 27 | host = proxy_item['ip'] |
| 28 | port = proxy_item['port'] |
| 29 | yield Proxy(host=host, port=port) |
| 30 | except json.JSONDecodeError: |
| 31 | print("json.JSONDecodeError") |
| 32 | return |
| 33 | |
| 34 | def crawl(self): |
| 35 | """ |