use proxy to crawl page :param url: page url :param proxy: proxy, such as 8.8.8.8:8888 :return: html
(url, proxy)
| 14 | |
| 15 | |
| 16 | def crawl(url, proxy): |
| 17 | """ |
| 18 | use proxy to crawl page |
| 19 | :param url: page url |
| 20 | :param proxy: proxy, such as 8.8.8.8:8888 |
| 21 | :return: html |
| 22 | """ |
| 23 | proxies = {'http': 'http://' + proxy} |
| 24 | return requests.get(url, proxies=proxies).text |
| 25 | |
| 26 | |
| 27 | def main(): |