(seed_url, match_pattern, *, max_depth=-1)
| 42 | |
| 43 | # 开始执行爬虫程序并对指定的数据进行持久化操作 |
| 44 | def start_crawl(seed_url, match_pattern, *, max_depth=-1): |
| 45 | conn = pymysql.connect(host='localhost', port=3306, |
| 46 | database='crawler', user='root', |
| 47 | password='123456', charset='utf8') |
| 48 | try: |
| 49 | with conn.cursor() as cursor: |
| 50 | url_list = [seed_url] |
| 51 | visited_url_list = {seed_url: 0} |
| 52 | while url_list: |
| 53 | current_url = url_list.pop(0) |
| 54 | depth = visited_url_list[current_url] |
| 55 | if depth != max_depth: |
| 56 | page_html = get_page_html(current_url, charsets=('utf-8', 'gbk', 'gb2312')) |
| 57 | links_list = get_matched_parts(page_html, match_pattern) |
| 58 | param_list = [] |
| 59 | for link in links_list: |
| 60 | if link not in visited_url_list: |
| 61 | visited_url_list[link] = depth + 1 |
| 62 | page_html = get_page_html(link, charsets=('utf-8', 'gbk', 'gb2312')) |
| 63 | headings = get_matched_parts(page_html, r'<h1>(.*)<span') |
| 64 | if headings: |
| 65 | param_list.append((headings[0], link)) |
| 66 | cursor.executemany('insert into tb_result values (default, %s, %s)', |
| 67 | param_list) |
| 68 | conn.commit() |
| 69 | except Error: |
| 70 | pass |
| 71 | # logging.error('SQL:', error) |
| 72 | finally: |
| 73 | conn.close() |
| 74 | |
| 75 | |
| 76 | def main(): |
no test coverage detected