(seed_url, match_pattern, *, max_depth=-1)
| 50 | |
| 51 | # 开始执行爬虫程序 |
| 52 | def start_crawl(seed_url, match_pattern, *, max_depth=-1): |
| 53 | client = redis.Redis(host='1.2.3.4', port=6379, password='1qaz2wsx') |
| 54 | charsets = ('utf-8', 'gbk', 'gb2312') |
| 55 | logging.info('[Redis ping]', client.ping()) |
| 56 | url_list = [seed_url] |
| 57 | visited_url_list = {seed_url: 0} |
| 58 | while url_list: |
| 59 | current_url = url_list.pop(0) |
| 60 | depth = visited_url_list[current_url] |
| 61 | if depth != max_depth: |
| 62 | page_html = get_page_html(current_url, charsets=charsets) |
| 63 | links_list = get_matched_parts(page_html, match_pattern) |
| 64 | for link in links_list: |
| 65 | if link not in visited_url_list: |
| 66 | visited_url_list[link] = depth + 1 |
| 67 | page_html = get_page_html(link, charsets=charsets) |
| 68 | if page_html: |
| 69 | hasher = hashlib.md5() |
| 70 | hasher.update(link.encode('utf-8')) |
| 71 | zipped_page = zlib.compress(pickle.dumps(page_html)) |
| 72 | client.set(hasher.hexdigest(), zipped_page) |
| 73 | |
| 74 | |
| 75 | def main(): |
no test coverage detected