Function prefetch_mirror

aura/prefetch.py:40–81

(uris: Iterable[str], workers=10)

def prefetch_mirror(uris: Iterable[str], workers=10):
    logger.info("Caching AST patterns")
    cache.ASTPatternCache.proxy()

    logger.info("Caching list of packages on pypi")
    cache.PyPIPackageList.proxy()

    #logger.info("Prefetching package JSON information")
    #lm = mirror.LocalMirror()
    #pkgs = tuple(lm.list_packages())
    #for x in tqdm.tqdm(pkgs, leave=False):
    #    lm.get_json(x)

    loop = asyncio.get_event_loop()

    pf = github.GitHubPrefetcher()
    loop.create_task(pf.process())

    uri_queue = AsyncQueue(desc="Mirror package prefetch")

    workers = [loop.create_task(fetch_package(uri_queue, pf)) for _ in range(workers)]

    for uri in uris:
        uri_queue.put_nowait(uri)

    logger.info("Waiting for prefetch workers to finish")

    loop.run_until_complete(uri_queue.join())

    # Cancel prefetch workers
    for worker in workers:
        worker.cancel()

    # Wait for all tasks to finish
    logger.info("All coroutines spawned, waiting for work to finish...")
    loop.run_until_complete(pf.queue.put(github.GitHubPrefetcher.STOP))

    pending = asyncio.all_tasks(loop=loop)
    loop.run_until_complete(asyncio.gather(*pending))

    loop.stop()
    loop.close()