Fallback fetch using urllib.
(url: str, timeout: int = 30)
| 180 | |
| 181 | |
| 182 | def fetch_with_urllib(url: str, timeout: int = 30) -> str: |
| 183 | """Fallback fetch using urllib.""" |
| 184 | ctx = ssl.create_default_context() |
| 185 | ctx.check_hostname = False |
| 186 | ctx.verify_mode = ssl.CERT_NONE |
| 187 | |
| 188 | headers = { |
| 189 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', |
| 190 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', |
| 191 | 'Accept-Language': 'en-US,en;q=0.5', |
| 192 | } |
| 193 | |
| 194 | request = urllib.request.Request(url, headers=headers) |
| 195 | |
| 196 | try: |
| 197 | with urllib.request.urlopen(request, timeout=timeout, context=ctx) as response: |
| 198 | content_type = response.headers.get('Content-Type', '') |
| 199 | encoding = 'utf-8' |
| 200 | if 'charset=' in content_type: |
| 201 | encoding = content_type.split('charset=')[-1].split(';')[0].strip() |
| 202 | return response.read().decode(encoding, errors='replace') |
| 203 | |
| 204 | except urllib.error.HTTPError as e: |
| 205 | return f"HTTP Error {e.code}: {e.reason}" |
| 206 | except urllib.error.URLError as e: |
| 207 | return f"URL Error: {e.reason}" |
| 208 | except Exception as e: |
| 209 | return f"Error fetching URL: {str(e)}" |
| 210 | |
| 211 | |
| 212 | def fetch_url(url: str, timeout: int = 30) -> str: |