Tracks progress and performance metrics for batch operations
| 19 | logger.info(f"{operation_name} completed in {duration:.2f}s") |
| 20 | |
| 21 | class ProgressTracker: |
| 22 | """Tracks progress and performance metrics for batch operations""" |
| 23 | |
| 24 | def __init__(self, total_items: int, operation_name: str = "Processing"): |
| 25 | self.total_items = total_items |
| 26 | self.operation_name = operation_name |
| 27 | self.processed_items = 0 |
| 28 | self.errors_encountered = 0 |
| 29 | self.start_time = time.time() |
| 30 | self.last_report_time = time.time() |
| 31 | self.report_interval = 10 # Report every 10 seconds |
| 32 | |
| 33 | def update(self, items_processed: int, errors: int = 0): |
| 34 | """Update progress with number of items processed""" |
| 35 | self.processed_items += items_processed |
| 36 | self.errors_encountered += errors |
| 37 | |
| 38 | current_time = time.time() |
| 39 | if current_time - self.last_report_time >= self.report_interval: |
| 40 | self._report_progress() |
| 41 | self.last_report_time = current_time |
| 42 | |
| 43 | def _report_progress(self): |
| 44 | """Report current progress""" |
| 45 | elapsed = time.time() - self.start_time |
| 46 | if elapsed > 0: |
| 47 | rate = self.processed_items / elapsed |
| 48 | remaining = self.total_items - self.processed_items |
| 49 | eta = remaining / rate if rate > 0 else 0 |
| 50 | |
| 51 | progress_pct = (self.processed_items / self.total_items) * 100 |
| 52 | |
| 53 | logger.info( |
| 54 | f"{self.operation_name}: {self.processed_items}/{self.total_items} " |
| 55 | f"({progress_pct:.1f}%) - {rate:.2f} items/sec - " |
| 56 | f"ETA: {eta/60:.1f}min - Errors: {self.errors_encountered}" |
| 57 | ) |
| 58 | |
| 59 | def finish(self): |
| 60 | """Report final statistics""" |
| 61 | elapsed = time.time() - self.start_time |
| 62 | rate = self.processed_items / elapsed if elapsed > 0 else 0 |
| 63 | |
| 64 | logger.info( |
| 65 | f"{self.operation_name} completed: {self.processed_items}/{self.total_items} items " |
| 66 | f"in {elapsed:.2f}s ({rate:.2f} items/sec) - {self.errors_encountered} errors" |
| 67 | ) |
| 68 | |
| 69 | class BatchProcessor: |
| 70 | """Generic batch processor with progress tracking and error handling""" |
no outgoing calls
no test coverage detected