(self, current_url, *, charsets=('utf-8', ),
user_agent=None, proxies=None)
| 60 | |
| 61 | @Retry() |
| 62 | def fetch(self, current_url, *, charsets=('utf-8', ), |
| 63 | user_agent=None, proxies=None): |
| 64 | thread_name = current_thread().name |
| 65 | print(f'[{thread_name}]: {current_url}') |
| 66 | headers = {'user-agent': user_agent} if user_agent else {} |
| 67 | resp = requests.get(current_url, |
| 68 | headers=headers, proxies=proxies) |
| 69 | return decode_page(resp.content, charsets) \ |
| 70 | if resp.status_code == 200 else None |
| 71 | |
| 72 | def parse(self, html_page, *, domain='m.sohu.com'): |
| 73 | soup = BeautifulSoup(html_page, 'lxml') |