| 51 | |
| 52 | @dataclass |
| 53 | class CrawlStats: |
| 54 | total: int = 0 |
| 55 | success: int = 0 |
| 56 | failures: int = 0 |
| 57 | retries: int = 0 |
| 58 | failure_reasons: Counter = field(default_factory=Counter) |
| 59 | retry_reasons: Counter = field(default_factory=Counter) |
| 60 | small_responses: list[ResponseInfo] = field(default_factory=list) |
| 61 | |
| 62 | def add_small_response(self, response_info: ResponseInfo): |
| 63 | """Add a small response to the tracking list, maintaining a reasonable size""" |
| 64 | self.small_responses.append(response_info) |
| 65 | # Keep the list at a reasonable size to avoid memory issues |
| 66 | if len(self.small_responses) > 100: # Keep the most recent 100 |
| 67 | self.small_responses = self.small_responses[-100:] |
| 68 | |
| 69 | def print_status(self): |
| 70 | """Print the current status""" |
| 71 | print(f"\rProgress: {self.success} successful | {self.failures} failed | " |
| 72 | f"Retries: {self.retries} | Total: {self.total}", end="", flush=True) |
| 73 | |
| 74 | def print_failure_summary(self): |
| 75 | """Print a summary of failures at the end of the crawl""" |
| 76 | print("\n\nFailure Analysis:") |
| 77 | print("-" * 60) |
| 78 | |
| 79 | # Print permanent failures |
| 80 | if self.failure_reasons: |
| 81 | print("Permanent Failures:") |
| 82 | for reason, count in self.failure_reasons.most_common(): |
| 83 | percentage = (count / self.failures * 100) if self.failures > 0 else 0 |
| 84 | print(f" {reason}: {count} ({percentage:.1f}%)") |
| 85 | else: |
| 86 | print("No permanent failures recorded") |
| 87 | |
| 88 | # Print retry statistics |
| 89 | if self.retry_reasons: |
| 90 | print("\nRetried Errors:") |
| 91 | total_retries = sum(self.retry_reasons.values()) |
| 92 | for reason, count in self.retry_reasons.most_common(): |
| 93 | percentage = (count / total_retries * 100) |
| 94 | print(f" {reason}: {count} ({percentage:.1f}%)") |
| 95 | else: |
| 96 | print("\nNo retries recorded") |
| 97 | |
| 98 | # Display small responses |
| 99 | if self.small_responses: |
| 100 | print("\nSamples of Small Responses:") |
| 101 | print("-" * 60) |
| 102 | # Show up to 10 samples |
| 103 | for i, response in enumerate(self.small_responses[-10:]): |
| 104 | print(f"{i+1}. {response}") |
| 105 | |
| 106 | print("-" * 60) |
| 107 | |
| 108 | class SimpleCrawler: |
| 109 | def __init__(self, target_dir: str, max_retries: int = 3): |