| 49 | |
| 50 | |
| 51 | class Spider(object): |
| 52 | |
| 53 | def __init__(self): |
| 54 | self.status = SpiderStatus.IDLE |
| 55 | |
| 56 | @Retry() |
| 57 | def fetch(self, current_url, *, charsets=('utf-8', ), |
| 58 | user_agent=None, proxies=None): |
| 59 | thread_name = current_thread().name |
| 60 | print(f'[{thread_name}]: {current_url}') |
| 61 | headers = {'user-agent': user_agent} if user_agent else {} |
| 62 | resp = requests.get(current_url, |
| 63 | headers=headers, proxies=proxies) |
| 64 | return decode_page(resp.content, charsets) \ |
| 65 | if resp.status_code == 200 else None |
| 66 | |
| 67 | def parse(self, html_page, *, domain='m.sohu.com'): |
| 68 | soup = BeautifulSoup(html_page, 'lxml') |
| 69 | url_links = [] |
| 70 | for a_tag in soup.body.select('a[href]'): |
| 71 | parser = urlparse(a_tag.attrs['href']) |
| 72 | scheme = parser.scheme or 'http' |
| 73 | netloc = parser.netloc or domain |
| 74 | if scheme != 'javascript' and netloc == domain: |
| 75 | path = parser.path |
| 76 | query = '?' + parser.query if parser.query else '' |
| 77 | full_url = f'{scheme}://{netloc}{path}{query}' |
| 78 | if full_url not in visited_urls: |
| 79 | url_links.append(full_url) |
| 80 | return url_links |
| 81 | |
| 82 | def extract(self, html_page): |
| 83 | pass |
| 84 | |
| 85 | def store(self, data_dict): |
| 86 | pass |
| 87 | |
| 88 | |
| 89 | class SpiderThread(Thread): |