| 126 | |
| 127 | |
| 128 | class BatchProgressTracker: |
| 129 | |
| 130 | def __init__(self): |
| 131 | self._total = 0 |
| 132 | self._completed = 0 |
| 133 | self._pbar: Optional[tqdm] = None |
| 134 | self._last_log_count = 0 |
| 135 | |
| 136 | def submitted(self): |
| 137 | self._total += 1 |
| 138 | |
| 139 | def completed(self): |
| 140 | self._completed += 1 |
| 141 | if self._pbar: |
| 142 | self._pbar.update() |
| 143 | |
| 144 | if self._total > 0: |
| 145 | log_interval = min(100, max(self._total // 10, 1)) |
| 146 | if self._completed - self._last_log_count >= log_interval: |
| 147 | console_logger.info(f"Progress: {self._completed}/{self._total} requests completed") |
| 148 | self._last_log_count = self._completed |
| 149 | |
| 150 | def pbar(self) -> tqdm: |
| 151 | self._pbar = tqdm( |
| 152 | total=self._total, |
| 153 | unit="req", |
| 154 | desc="Running batch", |
| 155 | mininterval=10, |
| 156 | bar_format=_BAR_FORMAT, |
| 157 | ) |
| 158 | return self._pbar |
| 159 | |
| 160 | |
| 161 | async def read_file(path_or_url: str) -> str: |
no outgoing calls