小舒代理 crawler, http://www.xsdaili.cn/
| 9 | |
| 10 | |
| 11 | class XiaoShuCrawler(BaseCrawler): |
| 12 | """ |
| 13 | 小舒代理 crawler, http://www.xsdaili.cn/ |
| 14 | """ |
| 15 | |
| 16 | def __init__(self): |
| 17 | """ |
| 18 | init urls |
| 19 | """ |
| 20 | try: |
| 21 | html = self.fetch(url=BASE_URL) |
| 22 | except: |
| 23 | self.urls = [] |
| 24 | return |
| 25 | doc = pq(html) |
| 26 | title = doc(".title:eq(0) a").items() |
| 27 | latest_page = 0 |
| 28 | for t in title: |
| 29 | res = re.search(r"/(\d+)\.html", t.attr("href")) |
| 30 | latest_page = int(res.group(1)) if res else 0 |
| 31 | if latest_page: |
| 32 | self.urls = [PAGE_BASE_URL.format(page=page) for page in range( |
| 33 | latest_page - MAX_PAGE, latest_page)] |
| 34 | else: |
| 35 | self.urls = [] |
| 36 | |
| 37 | def parse(self, html): |
| 38 | """ |
| 39 | parse html file to get proxies |
| 40 | :return: |
| 41 | """ |
| 42 | doc = pq(html) |
| 43 | contents = doc('.cont').text() |
| 44 | contents = contents.split("\n") |
| 45 | for content in contents: |
| 46 | c = content[:content.find("@")] |
| 47 | host, port = c.split(":") |
| 48 | yield Proxy(host=host, port=int(port)) |
| 49 | |
| 50 | |
| 51 | if __name__ == '__main__': |