Returns true if URL is of the "http" ("https") scheme. :param url: URL to test. :return: True if argument URL is of the "http" ("https") scheme.
(base_url, url: str)
| 138 | return extract_link_upto_nth_segment(0, url) + "sitemap.xml" |
| 139 | |
| 140 | def clean_url(base_url, url: str) -> bool: |
| 141 | """ |
| 142 | Returns true if URL is of the "http" ("https") scheme. |
| 143 | |
| 144 | :param url: URL to test. |
| 145 | :return: True if argument URL is of the "http" ("https") scheme. |
| 146 | """ |
| 147 | if url is None: |
| 148 | print("URL is None") |
| 149 | return False |
| 150 | if len(url) == 0: |
| 151 | print("URL is empty") |
| 152 | return False |
| 153 | |
| 154 | |
| 155 | try: |
| 156 | uri = urlparse(url) |
| 157 | _ = urlunparse(uri) |
| 158 | |
| 159 | except Exception as ex: |
| 160 | print(f"Cannot parse URL {url}: {ex}") |
| 161 | return False |
| 162 | |
| 163 | if not uri.scheme: |
| 164 | return join_link(base_url, url) |
| 165 | |
| 166 | if uri.scheme.lower() not in ["http", "https"]: |
| 167 | return join_link(base_url, url) |
| 168 | |
| 169 | if not uri.hostname: |
| 170 | return join_link(base_url, url) |
| 171 | |
| 172 | return url |
| 173 | |
| 174 | |
| 175 | def parse_sitemaps_from_robots_txt(base_url, robots_txt_content): |
no test coverage detected