Check if URL returns 200 and is not a generic redirect.
(cat, number, url)
| 20 | checked = 0 |
| 21 | |
| 22 | def check_url(cat, number, url): |
| 23 | """Check if URL returns 200 and is not a generic redirect.""" |
| 24 | try: |
| 25 | req = urllib.request.Request(url, method="HEAD", headers={ |
| 26 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)" |
| 27 | }) |
| 28 | resp = urllib.request.urlopen(req, timeout=15) |
| 29 | final_url = resp.geturl() |
| 30 | status = resp.status |
| 31 | # Check for redirect to generic page |
| 32 | if "search" in final_url or "products" in final_url: |
| 33 | return (cat, number, url, f"REDIRECT to generic: {final_url}") |
| 34 | return None |
| 35 | except urllib.error.HTTPError as e: |
| 36 | if e.code == 404: |
| 37 | return (cat, number, url, "404 Not Found") |
| 38 | elif e.code == 403: |
| 39 | return None # Some sites block HEAD, not necessarily an error |
| 40 | else: |
| 41 | return (cat, number, url, f"HTTP {e.code}") |
| 42 | except urllib.error.URLError as e: |
| 43 | return (cat, number, url, f"URL Error: {e.reason}") |
| 44 | except Exception as e: |
| 45 | return (cat, number, url, f"Error: {str(e)[:50]}") |
| 46 | |
| 47 | # Collect all URLs to check |
| 48 | urls_to_check = [] |
nothing calls this directly
no outgoing calls
no test coverage detected