Run extractor on each file sequentially. Callback exceptions are treated as fatal, just like unexpected extractor exceptions.
(
extractor: Extractor,
gz_files: list[str],
on_start: Callable[[str], None] | None = None,
on_result: Callable[[str, ExtractionResult], None] | None = None,
)
| 147 | |
| 148 | |
| 149 | def run_sequential( |
| 150 | extractor: Extractor, |
| 151 | gz_files: list[str], |
| 152 | on_start: Callable[[str], None] | None = None, |
| 153 | on_result: Callable[[str, ExtractionResult], None] | None = None, |
| 154 | ) -> BatchResult: |
| 155 | """Run extractor on each file sequentially. |
| 156 | |
| 157 | Callback exceptions are treated as fatal, just like unexpected extractor |
| 158 | exceptions. |
| 159 | """ |
| 160 | batch = BatchResult() |
| 161 | |
| 162 | for gz_path in gz_files: |
| 163 | try: |
| 164 | if on_start: |
| 165 | on_start(gz_path) |
| 166 | |
| 167 | entry = _extract_one(extractor, gz_path) |
| 168 | _tally(batch, entry) |
| 169 | if on_result: |
| 170 | on_result(gz_path, entry) |
| 171 | except KeyboardInterrupt: |
| 172 | logger.info("interrupted by user") |
| 173 | batch.interrupted = True |
| 174 | break |
| 175 | except FatalExtractionError: |
| 176 | raise |
| 177 | except Exception as e: |
| 178 | logger.exception( |
| 179 | "fatal unexpected exception in callback while processing %s", |
| 180 | gz_path, |
| 181 | ) |
| 182 | raise FatalExtractionError(str(e)) from e |
| 183 | |
| 184 | return batch |
| 185 | |
| 186 | |
| 187 | def run_parallel( |
no test coverage detected