(self, html_page, *, domain='m.sohu.com')
| 65 | if resp.status_code == 200 else None |
| 66 | |
| 67 | def parse(self, html_page, *, domain='m.sohu.com'): |
| 68 | soup = BeautifulSoup(html_page, 'lxml') |
| 69 | url_links = [] |
| 70 | for a_tag in soup.body.select('a[href]'): |
| 71 | parser = urlparse(a_tag.attrs['href']) |
| 72 | scheme = parser.scheme or 'http' |
| 73 | netloc = parser.netloc or domain |
| 74 | if scheme != 'javascript' and netloc == domain: |
| 75 | path = parser.path |
| 76 | query = '?' + parser.query if parser.query else '' |
| 77 | full_url = f'{scheme}://{netloc}{path}{query}' |
| 78 | if full_url not in visited_urls: |
| 79 | url_links.append(full_url) |
| 80 | return url_links |
| 81 | |
| 82 | def extract(self, html_page): |
| 83 | pass |
no outgoing calls
no test coverage detected