| 70 | if resp.status_code == 200 else None |
| 71 | |
| 72 | def parse(self, html_page, *, domain='m.sohu.com'): |
| 73 | soup = BeautifulSoup(html_page, 'lxml') |
| 74 | for a_tag in soup.body.select('a[href]'): |
| 75 | parser = urlparse(a_tag.attrs['href']) |
| 76 | scheme = parser.scheme or 'http' |
| 77 | netloc = parser.netloc or domain |
| 78 | if scheme != 'javascript' and netloc == domain: |
| 79 | path = parser.path |
| 80 | query = '?' + parser.query if parser.query else '' |
| 81 | full_url = f'{scheme}://{netloc}{path}{query}' |
| 82 | redis_client = thread_local.redis_client |
| 83 | if not redis_client.sismember('visited_urls', full_url): |
| 84 | redis_client.rpush('m_sohu_task', full_url) |
| 85 | |
| 86 | def extract(self, html_page): |
| 87 | pass |