(
self, conv_input: _DocumentConversionInput, raises_on_error: bool
)
| 241 | ) |
| 242 | |
| 243 | def _convert( |
| 244 | self, conv_input: _DocumentConversionInput, raises_on_error: bool |
| 245 | ) -> Iterator[ConversionResult]: |
| 246 | start_time = time.monotonic() |
| 247 | |
| 248 | for input_batch in chunkify( |
| 249 | conv_input.docs(self.format_to_options), |
| 250 | settings.perf.doc_batch_size, # pass format_options |
| 251 | ): |
| 252 | _log.info(f"Going to convert document batch...") |
| 253 | |
| 254 | # parallel processing only within input_batch |
| 255 | # with ThreadPoolExecutor( |
| 256 | # max_workers=settings.perf.doc_batch_concurrency |
| 257 | # ) as pool: |
| 258 | # yield from pool.map(self.process_document, input_batch) |
| 259 | # Note: PDF backends are not thread-safe, thread pool usage was disabled. |
| 260 | |
| 261 | for item in map( |
| 262 | partial(self._process_document, raises_on_error=raises_on_error), |
| 263 | input_batch, |
| 264 | ): |
| 265 | elapsed = time.monotonic() - start_time |
| 266 | start_time = time.monotonic() |
| 267 | _log.info( |
| 268 | f"Finished converting document {item.input.file.name} in {elapsed:.2f} sec." |
| 269 | ) |
| 270 | yield item |
| 271 | |
| 272 | def _get_pipeline(self, doc_format: InputFormat) -> Optional[BasePipeline]: |
| 273 | fopt = self.format_to_options.get(doc_format) |
no test coverage detected