(
html,
stop_urls=(
"javascript",
"+",
".css",
".js",
".rar",
".xls",
".exe",
".apk",
".doc",
".jpg",
".png",
".flv",
".mp4",
),
)
| 419 | |
| 420 | |
| 421 | def get_urls( |
| 422 | html, |
| 423 | stop_urls=( |
| 424 | "javascript", |
| 425 | "+", |
| 426 | ".css", |
| 427 | ".js", |
| 428 | ".rar", |
| 429 | ".xls", |
| 430 | ".exe", |
| 431 | ".apk", |
| 432 | ".doc", |
| 433 | ".jpg", |
| 434 | ".png", |
| 435 | ".flv", |
| 436 | ".mp4", |
| 437 | ), |
| 438 | ): |
| 439 | # 不匹配javascript、 +、 # 这样的url |
| 440 | regex = r'<a.*?href.*?=.*?["|\'](.*?)["|\']' |
| 441 | |
| 442 | urls = get_info(html, regex) |
| 443 | urls = sorted(set(urls), key=urls.index) |
| 444 | if stop_urls: |
| 445 | stop_urls = isinstance(stop_urls, str) and [stop_urls] or stop_urls |
| 446 | use_urls = [] |
| 447 | for url in urls: |
| 448 | for stop_url in stop_urls: |
| 449 | if stop_url in url: |
| 450 | break |
| 451 | else: |
| 452 | use_urls.append(url) |
| 453 | |
| 454 | urls = use_urls |
| 455 | return urls |
| 456 | |
| 457 | |
| 458 | def get_full_url(root_url, sub_url): |
nothing calls this directly
no test coverage detected