Return a list of link URLs found in the HTML page at `self.url`
(self, _CACHE=[])
| 1747 | self.links = self.find_links() |
| 1748 | |
| 1749 | def find_links(self, _CACHE=[]): |
| 1750 | """ |
| 1751 | Return a list of link URLs found in the HTML page at `self.url` |
| 1752 | """ |
| 1753 | if _CACHE: |
| 1754 | return _CACHE |
| 1755 | |
| 1756 | links_url = self.url |
| 1757 | if TRACE_DEEP: |
| 1758 | print(f"Finding links from: {links_url}") |
| 1759 | plinks_url = urllib.parse.urlparse(links_url) |
| 1760 | base_url = urllib.parse.SplitResult( |
| 1761 | plinks_url.scheme, plinks_url.netloc, "", "", "" |
| 1762 | ).geturl() |
| 1763 | |
| 1764 | if TRACE_DEEP: |
| 1765 | print(f"Base URL {base_url}") |
| 1766 | |
| 1767 | text = CACHE.get( |
| 1768 | path_or_url=links_url, |
| 1769 | as_text=True, |
| 1770 | force=not self.use_cached_index, |
| 1771 | ) |
| 1772 | |
| 1773 | links = [] |
| 1774 | for link in collect_urls(text): |
| 1775 | if not link.endswith(EXTENSIONS): |
| 1776 | continue |
| 1777 | |
| 1778 | plink = urllib.parse.urlsplit(link) |
| 1779 | |
| 1780 | if plink.scheme: |
| 1781 | # full URL kept as-is |
| 1782 | url = link |
| 1783 | |
| 1784 | if plink.path.startswith("/"): |
| 1785 | # absolute link |
| 1786 | url = f"{base_url}{link}" |
| 1787 | |
| 1788 | else: |
| 1789 | # relative link |
| 1790 | url = f"{links_url}/{link}" |
| 1791 | |
| 1792 | if TRACE_DEEP: |
| 1793 | print(f"Adding URL: {url}") |
| 1794 | |
| 1795 | links.append(url) |
| 1796 | |
| 1797 | if TRACE: |
| 1798 | print(f"Found {len(links)} links at {links_url}") |
| 1799 | _CACHE.extend(links) |
| 1800 | return links |
| 1801 | |
| 1802 | @classmethod |
| 1803 | def from_url(cls, url=ABOUT_BASE_URL, _LINKS_REPO={}, use_cached_index=False): |
no test coverage detected