Function download_links

utils/src/experiment/crawler.py:181–216

(jsonl_file: str)

async def download_links(jsonl_file: str) -> None:
    async with aiohttp.ClientSession() as session:
        with jsonlines.open(jsonl_file) as reader:
            tasks = list(reader)
        progress_bar = tqdm(total=len(tasks), desc="Downloading files")
        task_iter = iter(tasks)
        coroutines = []
        while True:
            while len(coroutines) < 80:
                task = next(task_iter, None)
                if task is None:
                    break
                dirname = f"zenodo-pptx/pptx/{task['license']}/{task['created'][:4]}/"
                basename = f"{task['checksum'][4:]}-{task['filename']}"
                filepath = dirname + basename
                try:
                    open("/tmp/" + basename, "wb").close()
                except:
                    filepath = dirname + basename[:240] + ".pptx"
                if os.path.exists(filepath):
                    progress_bar.update(1)
                    continue
                coroutines.append(
                    download_file(session, filepath, task["url"], progress_bar)
                )

            start_time = time.time()
            results = await asyncio.gather(*coroutines, return_exceptions=True)
            for result in results:
                if isinstance(result, Exception):
                    tqdm.write(f"Error {result}")
            if len(coroutines) % 80 != 0:
                return
            coroutines.clear()
            elapsed_time = time.time() - start_time
            sleep(max(60 - elapsed_time, 0))


async def gather_files(topics: list[str], num_results: int) -> None: