Function yield_all_page_url

spiderFile/get_tj_accident_info.py:8–19

生成所有的页面url @param root_url: 首页url type root_url: str @param page: 爬取的页面个数 type page: int

(root_url, page=51)

from bs4 import BeautifulSoup

def yield_all_page_url(root_url, page=51):
    """生成所有的页面url
    @param root_url: 首页url
    type root_url: str
    @param page: 爬取的页面个数
    type page: int
    """
    # 观察网站翻页结构可知
    page_url_list = [f"{root_url}index_{i}.html" for i in range(1, page)]
    # 添加首页url
    page_url_list.insert(0, root_url)
    return page_url_list

async def get_info_page_url(url, session):
    regex = re.compile("<a href='./(.*?)'\s+title=")