Uqidata crawler, https://ip.uqidata.com/free/index.html
| 7 | |
| 8 | |
| 9 | class UqidataCrawler(BaseCrawler): |
| 10 | """ |
| 11 | Uqidata crawler, https://ip.uqidata.com/free/index.html |
| 12 | """ |
| 13 | urls = [BASE_URL] |
| 14 | ignore = True |
| 15 | |
| 16 | def encode(input_str): |
| 17 | tmp = [] |
| 18 | for i in range(len(input_str)): |
| 19 | tmp.append("ABCDEFGHIZ".find(input_str[i])) |
| 20 | result = "".join(str(i) for i in tmp) |
| 21 | result = int(result) >> 0x03 |
| 22 | return result |
| 23 | |
| 24 | def parse(self, html): |
| 25 | """ |
| 26 | parse html file to get proxies |
| 27 | :return: |
| 28 | """ |
| 29 | doc = pq(html) |
| 30 | trs = doc('#main_container .inner table tbody tr:nth-child(n+3)').items() |
| 31 | for tr in trs: |
| 32 | ip_html = tr('td.ip').find("*").items() |
| 33 | host = '' |
| 34 | for i in ip_html: |
| 35 | if i.attr('style') is not None and 'none' in i.attr('style'): |
| 36 | continue |
| 37 | if i.text() == '': |
| 38 | continue |
| 39 | host += i.text() |
| 40 | |
| 41 | port_code = tr('td.port').attr('class').split(' ')[1] |
| 42 | port = UqidataCrawler.encode(port_code) |
| 43 | yield Proxy(host=host, port=port) |
| 44 | |
| 45 | |
| 46 | if __name__ == '__main__': |