(self, current_url, *, charsets=('utf-8', ),
user_agent=None, proxies=None)
| 55 | |
| 56 | @Retry() |
| 57 | def fetch(self, current_url, *, charsets=('utf-8', ), |
| 58 | user_agent=None, proxies=None): |
| 59 | thread_name = current_thread().name |
| 60 | print(f'[{thread_name}]: {current_url}') |
| 61 | headers = {'user-agent': user_agent} if user_agent else {} |
| 62 | resp = requests.get(current_url, |
| 63 | headers=headers, proxies=proxies) |
| 64 | return decode_page(resp.content, charsets) \ |
| 65 | if resp.status_code == 200 else None |
| 66 | |
| 67 | def parse(self, html_page, *, domain='m.sohu.com'): |
| 68 | soup = BeautifulSoup(html_page, 'lxml') |