(self, inner_paths, pool_size=100, only_if_bad=False)
| 331 | return valid |
| 332 | |
| 333 | def pooledDownloadContent(self, inner_paths, pool_size=100, only_if_bad=False): |
| 334 | self.log.debug("New downloadContent pool: len: %s, only if bad: %s" % (len(inner_paths), only_if_bad)) |
| 335 | self.worker_manager.started_task_num += len(inner_paths) |
| 336 | pool = gevent.pool.Pool(pool_size) |
| 337 | num_skipped = 0 |
| 338 | site_size_limit = self.getSizeLimit() * 1024 * 1024 |
| 339 | for inner_path in inner_paths: |
| 340 | if not only_if_bad or inner_path in self.bad_files: |
| 341 | pool.spawn(self.downloadContent, inner_path) |
| 342 | else: |
| 343 | num_skipped += 1 |
| 344 | self.worker_manager.started_task_num -= 1 |
| 345 | if self.settings["size"] > site_size_limit * 0.95: |
| 346 | self.log.warning("Site size limit almost reached, aborting downloadContent pool") |
| 347 | for aborted_inner_path in inner_paths: |
| 348 | if aborted_inner_path in self.bad_files: |
| 349 | del self.bad_files[aborted_inner_path] |
| 350 | self.worker_manager.removeSolvedFileTasks(mark_as_good=False) |
| 351 | break |
| 352 | self.log.debug("Ended downloadContent pool len: %s, skipped: %s" % (len(inner_paths), num_skipped)) |
| 353 | |
| 354 | def pooledDownloadFile(self, inner_paths, pool_size=100, only_if_bad=False): |
| 355 | self.log.debug("New downloadFile pool: len: %s, only if bad: %s" % (len(inner_paths), only_if_bad)) |
no test coverage detected