| 7 | |
| 8 | |
| 9 | def main(): |
| 10 | headers = {'user-agent': 'Baiduspider'} |
| 11 | proxies = { |
| 12 | 'http': 'http://122.114.31.177:808' |
| 13 | } |
| 14 | base_url = 'https://www.zhihu.com/' |
| 15 | seed_url = urljoin(base_url, 'explore') |
| 16 | resp = requests.get(seed_url, |
| 17 | headers=headers, |
| 18 | proxies=proxies) |
| 19 | soup = BeautifulSoup(resp.text, 'lxml') |
| 20 | href_regex = re.compile(r'^/question') |
| 21 | link_set = set() |
| 22 | for a_tag in soup.find_all('a', {'href': href_regex}): |
| 23 | if 'href' in a_tag.attrs: |
| 24 | href = a_tag.attrs['href'] |
| 25 | full_url = urljoin(base_url, href) |
| 26 | link_set.add(full_url) |
| 27 | print('Total %d question pages found.' % len(link_set)) |
| 28 | |
| 29 | |
| 30 | if __name__ == '__main__': |