| 12 | |
| 13 | |
| 14 | def main(): |
| 15 | # 指定种子页面 |
| 16 | base_url = 'https://www.zhihu.com/' |
| 17 | seed_url = urljoin(base_url, 'explore') |
| 18 | # 创建Redis客户端 |
| 19 | client = Redis(host='1.2.3.4', port=6379, password='1qaz2wsx') |
| 20 | # 设置用户代理(否则访问会被拒绝) |
| 21 | headers = {'user-agent': 'Baiduspider'} |
| 22 | # 通过requests模块发送GET请求并指定用户代理 |
| 23 | resp = requests.get(seed_url, headers=headers) |
| 24 | # 创建BeautifulSoup对象并指定使用lxml作为解析器 |
| 25 | soup = BeautifulSoup(resp.text, 'lxml') |
| 26 | href_regex = re.compile(r'^/question') |
| 27 | # 将URL处理成SHA1摘要(长度固定更简短) |
| 28 | hasher_proto = sha1() |
| 29 | # 查找所有href属性以/question打头的a标签 |
| 30 | for a_tag in soup.find_all('a', {'href': href_regex}): |
| 31 | # 获取a标签的href属性值并组装完整的URL |
| 32 | href = a_tag.attrs['href'] |
| 33 | full_url = urljoin(base_url, href) |
| 34 | # 传入URL生成SHA1摘要 |
| 35 | hasher = hasher_proto.copy() |
| 36 | hasher.update(full_url.encode('utf-8')) |
| 37 | field_key = hasher.hexdigest() |
| 38 | # 如果Redis的键'zhihu'对应的hash数据类型中没有URL的摘要就访问页面并缓存 |
| 39 | if not client.hexists('zhihu', field_key): |
| 40 | html_page = requests.get(full_url, headers=headers).text |
| 41 | # 对页面进行序列化和压缩操作 |
| 42 | zipped_page = zlib.compress(pickle.dumps(html_page)) |
| 43 | # 使用hash数据类型保存URL摘要及其对应的页面代码 |
| 44 | client.hset('zhihu', field_key, zipped_page) |
| 45 | # 显示总共缓存了多少个页面 |
| 46 | print('Total %d question pages found.' % client.hlen('zhihu')) |
| 47 | |
| 48 | |
| 49 | if __name__ == '__main__': |