| 23 | |
| 24 | |
| 25 | class CrawlThread(threading.Thread): |
| 26 | def __init__(self, proxyip): |
| 27 | super(CrawlThread, self).__init__() |
| 28 | self.proxyip = proxyip |
| 29 | |
| 30 | def run(self): |
| 31 | # 开始计时 |
| 32 | pure_ip_address = self.proxyip.split(':')[0] |
| 33 | # 验证IP归属 |
| 34 | if not getChinaIP(pure_ip_address): |
| 35 | # pass |
| 36 | raise ValueError('不是有效IP') |
| 37 | # |
| 38 | start = time.time() |
| 39 | # 消除关闭证书验证的警告 |
| 40 | urllib3.disable_warnings() |
| 41 | headers = Headers(headers=True).generate() |
| 42 | headers['Referer'] = 'http://bb.cf08tp.cn/Home/index.php?m=Index&a=index&id=2676' |
| 43 | headers['Pragma'] = 'no-cache' |
| 44 | headers['Host'] = 'bb.cf08tp.cn' |
| 45 | headers['x-forward-for'] = pure_ip_address |
| 46 | headers['Cookie'] = 'PHPSESSID={}'.format( |
| 47 | ''.join(str(uuid.uuid1()).split('-'))) |
| 48 | print(headers) |
| 49 | html = requests.get(headers=headers, url=targetUrl, proxies={ |
| 50 | "http": 'http://' + self.proxyip, "https": 'https://' + self.proxyip}, verify=False, timeout=2).content.decode() |
| 51 | # 结束计时 |
| 52 | end = time.time() |
| 53 | # 输出内容 |
| 54 | print(threading.current_thread().getName() + "使用代理IP, 耗时 " + str(end - start) + |
| 55 | "毫秒 " + self.proxyip + " 获取到如下HTML内容：\n" + html + "\n*************") |
| 56 | |
| 57 | # 获取代理IP的线程类 |
| 58 | |