(uri_queue, github_prefetcher)
| 16 | |
| 17 | |
| 18 | async def fetch_package(uri_queue, github_prefetcher): |
| 19 | try: |
| 20 | while True: |
| 21 | uri = await uri_queue.get() |
| 22 | try: |
| 23 | logger.info(f"Prefetching: `{uri}`") |
| 24 | handler = await non_blocking(URIHandler.from_uri, uri) |
| 25 | for x in await non_blocking(handler.get_paths): |
| 26 | if pkg := x.metadata.get("package_instance"): |
| 27 | source_url = pkg.source_url |
| 28 | if source_url: |
| 29 | await github_prefetcher.queue.put(source_url) |
| 30 | except NoSuchPackage: |
| 31 | logger.info(f"Package does not exists: `{uri}`") |
| 32 | except Exception: |
| 33 | logger.exception(f"An error occurred while prefetching the uri: `{uri}`") |
| 34 | finally: |
| 35 | uri_queue.task_done() |
| 36 | except asyncio.CancelledError: |
| 37 | pass |
| 38 | |
| 39 | |
| 40 | def prefetch_mirror(uris: Iterable[str], workers=10): |
no test coverage detected