(source, file_types, limit, all_files, output_dir, visited_urls=set(), max_depth=1, depth=0)
| 465 | |
| 466 | |
| 467 | def download_from_source(source, file_types, limit, all_files, output_dir, visited_urls=set(), max_depth=1, depth=0): |
| 468 | if depth > max_depth or source in visited_urls: |
| 469 | return |
| 470 | |
| 471 | print("IN DOWNLOAD FROM SOURCE") |
| 472 | |
| 473 | visited_urls.add(source) |
| 474 | prGreen(f"Crawling: {source} (Depth {depth})") |
| 475 | |
| 476 | file_links = get_file_links(source, file_types, limit, all_files) |
| 477 | for file_url in file_links: |
| 478 | download_file(file_url, output_dir) |
| 479 | |
| 480 | # Extract and follow links recursively |
| 481 | try: |
| 482 | response = requests.get(source, timeout=5) |
| 483 | response.raise_for_status() |
| 484 | soup = BeautifulSoup(response.text, 'html.parser') |
| 485 | for link in soup.find_all('a', href=True): |
| 486 | href = link['href'] |
| 487 | full_url = urljoin(source, href) |
| 488 | if is_valid_url(full_url) and full_url not in visited_urls: |
| 489 | download_from_source(full_url, file_types, limit, all_files, output_dir, visited_urls, max_depth, |
| 490 | depth + 1) |
| 491 | except requests.RequestException as e: |
| 492 | prRed(f"Failed to crawl {source}: {e}") |
| 493 | |
| 494 | |
| 495 | def threaded_download(source, *args): |
no test coverage detected