| 106 | print("-" * 60) |
| 107 | |
| 108 | class SimpleCrawler: |
| 109 | def __init__(self, target_dir: str, max_retries: int = 3): |
| 110 | self.target_dir = target_dir |
| 111 | self.max_retries = max_retries |
| 112 | self.stats = CrawlStats() |
| 113 | |
| 114 | # Create target directory if it doesn't exist |
| 115 | os.makedirs(target_dir, exist_ok=True) |
| 116 | |
| 117 | def categorize_error(self, error, status: int | None = None, content_length: int | None = None) -> str: |
| 118 | """Categorize the type of error encountered""" |
| 119 | error_str = str(error).lower() |
| 120 | |
| 121 | if content_length is not None and content_length < 1024: |
| 122 | return "Content too small (<1KB)" |
| 123 | elif status is not None: |
| 124 | if status == 403: |
| 125 | return "Access forbidden (403)" |
| 126 | elif status == 404: |
| 127 | return "Page not found (404)" |
| 128 | elif status == 429: |
| 129 | return "Rate limited (429)" |
| 130 | elif status == 500: |
| 131 | return "Server error (500)" |
| 132 | elif status >= 400: |
| 133 | return f"HTTP error {status}" |
| 134 | |
| 135 | if "timeout" in error_str: |
| 136 | return "Request timeout" |
| 137 | elif "connection" in error_str: |
| 138 | return "Connection error" |
| 139 | elif "dns" in error_str: |
| 140 | return "DNS resolution failed" |
| 141 | elif "ssl" in error_str: |
| 142 | return "SSL/TLS error" |
| 143 | elif "invalid" in error_str and "url" in error_str: |
| 144 | return "Invalid URL" |
| 145 | return f"Other error: {error_str[:100]}" |
| 146 | |
| 147 | def get_retry_delay(self, attempt: int, error_type: str | None = None) -> float: |
| 148 | """ |
| 149 | Calculate delay using exponential backoff algorithm. |
| 150 | Base delay is 2 seconds, doubled for each attempt with some jitter. |
| 151 | """ |
| 152 | # Longer base delay for rate limiting errors |
| 153 | base_delay = 5.0 if error_type and "429" in error_type else 2.0 |
| 154 | |
| 155 | # Calculate exponential backoff with jitter |
| 156 | # 2^attempt gives us: 2, 4, 8, 16, 32, etc. |
| 157 | delay = base_delay * (2 ** attempt) |
| 158 | |
| 159 | # Add jitter (±25%) |
| 160 | jitter = delay * 0.25 |
| 161 | delay = delay + random.uniform(-jitter, jitter) |
| 162 | |
| 163 | # Cap maximum delay at 5 minutes |
| 164 | return min(delay, 300) |
| 165 |
no outgoing calls