| 54 | |
| 55 | |
| 56 | class Spider(object): |
| 57 | |
| 58 | def __init__(self): |
| 59 | self.status = SpiderStatus.IDLE |
| 60 | |
| 61 | @Retry() |
| 62 | def fetch(self, current_url, *, charsets=('utf-8', ), |
| 63 | user_agent=None, proxies=None): |
| 64 | thread_name = current_thread().name |
| 65 | print(f'[{thread_name}]: {current_url}') |
| 66 | headers = {'user-agent': user_agent} if user_agent else {} |
| 67 | resp = requests.get(current_url, |
| 68 | headers=headers, proxies=proxies) |
| 69 | return decode_page(resp.content, charsets) \ |
| 70 | if resp.status_code == 200 else None |
| 71 | |
| 72 | def parse(self, html_page, *, domain='m.sohu.com'): |
| 73 | soup = BeautifulSoup(html_page, 'lxml') |
| 74 | for a_tag in soup.body.select('a[href]'): |
| 75 | parser = urlparse(a_tag.attrs['href']) |
| 76 | scheme = parser.scheme or 'http' |
| 77 | netloc = parser.netloc or domain |
| 78 | if scheme != 'javascript' and netloc == domain: |
| 79 | path = parser.path |
| 80 | query = '?' + parser.query if parser.query else '' |
| 81 | full_url = f'{scheme}://{netloc}{path}{query}' |
| 82 | redis_client = thread_local.redis_client |
| 83 | if not redis_client.sismember('visited_urls', full_url): |
| 84 | redis_client.rpush('m_sohu_task', full_url) |
| 85 | |
| 86 | def extract(self, html_page): |
| 87 | pass |
| 88 | |
| 89 | def store(self, data_dict): |
| 90 | # redis_client = thread_local.redis_client |
| 91 | # mongo_db = thread_local.mongo_db |
| 92 | pass |
| 93 | |
| 94 | |
| 95 | class SpiderThread(Thread): |