(url, file_types, limit, all_files)
| 398 | |
| 399 | |
| 400 | def get_file_links(url, file_types, limit, all_files): |
| 401 | valid_extensions = {"png", "jpg", "jpeg", "pdf", "docx", "mp3", "wav", "bin"} |
| 402 | try: |
| 403 | response = requests.get(url, timeout=5) |
| 404 | response.raise_for_status() |
| 405 | soup = BeautifulSoup(response.text, 'html.parser') |
| 406 | links = set() |
| 407 | |
| 408 | for link in soup.find_all('a', href=True): |
| 409 | href = link['href'].strip() |
| 410 | full_url = urljoin(url, href) |
| 411 | |
| 412 | if not is_valid_url(full_url): |
| 413 | continue # Skip invalid URLs |
| 414 | |
| 415 | parsed_href = urlparse(full_url).path |
| 416 | file_ext = parsed_href.split('.')[-1].lower() |
| 417 | |
| 418 | if all_files: |
| 419 | if file_ext in valid_extensions or ('wikipedia.org' in full_url and '/wiki/File:' in full_url): |
| 420 | links.add(full_url) |
| 421 | elif file_ext in file_types: |
| 422 | links.add(full_url) |
| 423 | if len(links) >= limit: |
| 424 | break |
| 425 | |
| 426 | return list(links)[:limit] if not all_files else list(links) |
| 427 | except requests.RequestException as e: |
| 428 | prRed(f"Failed to fetch links from {url}: {e}") |
| 429 | return [] |
| 430 | |
| 431 | |
| 432 | def copy_local_files(source, destination): |
no test coverage detected