| 4 | |
| 5 | |
| 6 | class TiebaSpider(): |
| 7 | |
| 8 | def __init__(self, kw, max_pn): |
| 9 | self.max_pn = max_pn |
| 10 | self.kw = kw |
| 11 | self.base_url = "https://tieba.baidu.com/f?kw={}&ie=utf-8&pn={}" |
| 12 | self.headers = { |
| 13 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" |
| 14 | } |
| 15 | pass |
| 16 | |
| 17 | def get_url_list(self): |
| 18 | ''' |
| 19 | 获取 url 列表 |
| 20 | :return: |
| 21 | ''' |
| 22 | # 写法一 |
| 23 | ''' |
| 24 | url_list = [] |
| 25 | |
| 26 | for pn in range(0,self.max_pn,50): |
| 27 | url = self.base_url.format(self.kw,pn) |
| 28 | url_list.append(url) |
| 29 | |
| 30 | return url_list |
| 31 | ''' |
| 32 | # 写法二 |
| 33 | return [self.base_url.format(self.kw, pn) for pn in range(0, self.max_pn, 50)] |
| 34 | |
| 35 | def get_content(self, url): |
| 36 | ''' |
| 37 | 发送请求获取响应内容 |
| 38 | :param url: |
| 39 | :return: |
| 40 | ''' |
| 41 | response = requests.get( |
| 42 | url=url, |
| 43 | headers=self.headers |
| 44 | ) |
| 45 | |
| 46 | return response.content |
| 47 | |
| 48 | def get_items(self, content, index): |
| 49 | ''' |
| 50 | 从响应内容中提取数据 |
| 51 | :param content: |
| 52 | :return: |
| 53 | ''' |
| 54 | with open('tieba-{}.html'.format(index), 'wb') as f: |
| 55 | f.write(content) |
| 56 | return None |
| 57 | |
| 58 | def save_items(self, items): |
| 59 | ''' |
| 60 | 保存数据 |
| 61 | :param items: |
| 62 | :return: |
| 63 | ''' |