| 17 | 2017-05-27 |
| 18 | """ |
| 19 | def get_proxys(page = 1): |
| 20 | #requests的Session可以自动保持cookie,不需要自己维护cookie内容 |
| 21 | S = requests.Session() |
| 22 | #西祠代理高匿IP地址 |
| 23 | target_url = 'http://www.xicidaili.com/nn/%d' % page |
| 24 | #完善的headers |
| 25 | target_headers = {'Upgrade-Insecure-Requests':'1', |
| 26 | 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36', |
| 27 | 'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', |
| 28 | 'Referer':'http://www.xicidaili.com/nn/', |
| 29 | 'Accept-Encoding':'gzip, deflate, sdch', |
| 30 | 'Accept-Language':'zh-CN,zh;q=0.8', |
| 31 | } |
| 32 | #get请求 |
| 33 | target_response = S.get(url = target_url, headers = target_headers) |
| 34 | #utf-8编码 |
| 35 | target_response.encoding = 'utf-8' |
| 36 | #获取网页信息 |
| 37 | target_html = target_response.text |
| 38 | #获取id为ip_list的table |
| 39 | bf1_ip_list = BeautifulSoup(target_html, 'lxml') |
| 40 | bf2_ip_list = BeautifulSoup(str(bf1_ip_list.find_all(id = 'ip_list')), 'lxml') |
| 41 | ip_list_info = bf2_ip_list.table.contents |
| 42 | #存储代理的列表 |
| 43 | proxys_list = [] |
| 44 | #爬取每个代理信息 |
| 45 | for index in range(len(ip_list_info)): |
| 46 | if index % 2 == 1 and index != 1: |
| 47 | dom = etree.HTML(str(ip_list_info[index])) |
| 48 | ip = dom.xpath('//td[2]') |
| 49 | port = dom.xpath('//td[3]') |
| 50 | protocol = dom.xpath('//td[6]') |
| 51 | proxys_list.append(protocol[0].text.lower() + '#' + ip[0].text + '#' + port[0].text) |
| 52 | #返回代理列表 |
| 53 | return proxys_list |
| 54 | |
| 55 | """ |
| 56 | 函数说明:检查代理IP的连通性 |