Categorize the type of error encountered
(self, error, status: int | None = None, content_length: int | None = None)
| 115 | os.makedirs(target_dir, exist_ok=True) |
| 116 | |
| 117 | def categorize_error(self, error, status: int | None = None, content_length: int | None = None) -> str: |
| 118 | """Categorize the type of error encountered""" |
| 119 | error_str = str(error).lower() |
| 120 | |
| 121 | if content_length is not None and content_length < 1024: |
| 122 | return "Content too small (<1KB)" |
| 123 | elif status is not None: |
| 124 | if status == 403: |
| 125 | return "Access forbidden (403)" |
| 126 | elif status == 404: |
| 127 | return "Page not found (404)" |
| 128 | elif status == 429: |
| 129 | return "Rate limited (429)" |
| 130 | elif status == 500: |
| 131 | return "Server error (500)" |
| 132 | elif status >= 400: |
| 133 | return f"HTTP error {status}" |
| 134 | |
| 135 | if "timeout" in error_str: |
| 136 | return "Request timeout" |
| 137 | elif "connection" in error_str: |
| 138 | return "Connection error" |
| 139 | elif "dns" in error_str: |
| 140 | return "DNS resolution failed" |
| 141 | elif "ssl" in error_str: |
| 142 | return "SSL/TLS error" |
| 143 | elif "invalid" in error_str and "url" in error_str: |
| 144 | return "Invalid URL" |
| 145 | return f"Other error: {error_str[:100]}" |
| 146 | |
| 147 | def get_retry_delay(self, attempt: int, error_type: str | None = None) -> float: |
| 148 | """ |