(topics: list[str], num_results: int)
| 217 | |
| 218 | |
| 219 | async def gather_files(topics: list[str], num_results: int) -> None: |
| 220 | session = aiohttp.ClientSession() |
| 221 | filetypes = ["pptx", "pdf"] |
| 222 | progress_bar = tqdm( |
| 223 | total=len(topics) * len(filetypes) * num_results, desc="Gathering files" |
| 224 | ) |
| 225 | writer = jsonlines.open(f"data/datastats.jsonl", mode="a") |
| 226 | existed = defaultdict(list) |
| 227 | for i in jsonlines.open(f"data/datastats.jsonl"): |
| 228 | existed[i["topic"] + i["filetype"]].append(i) |
| 229 | for topic, filetype in product(topics, filetypes): |
| 230 | selected = existed.get(topic + filetype, []) |
| 231 | progress_bar.set_description(f"Gathering {topic} {filetype}") |
| 232 | progress_bar.update(len(selected)) |
| 233 | page = 1 if len(selected) == 0 else selected[-1]["page"] + 1 |
| 234 | for record, page in iter_zenodo( |
| 235 | query=topic, filetype=filetype, sort="mostviewed", page=page |
| 236 | ): |
| 237 | if len(selected) >= num_results: |
| 238 | break |
| 239 | license = record["metadata"].get("license", {"id": "unknown"})["id"] |
| 240 | if license in BANNED_LICENSES: |
| 241 | continue |
| 242 | for file in record["files"]: |
| 243 | if not file["key"].endswith(f".{filetype}"): |
| 244 | continue |
| 245 | filepath = f"zenodo-pptx/{filetype}/{license}/{record['created'][:4]}/{file['checksum'][4:]}-{file['key']}" |
| 246 | dst = f"data/{topic}/{filetype}/{file['key'].rsplit('.')[0]}/original.{filetype}" |
| 247 | if os.path.exists(dst): |
| 248 | continue |
| 249 | os.makedirs(os.path.dirname(dst)) |
| 250 | if os.path.exists(filepath): |
| 251 | shutil.copy(filepath, dst) |
| 252 | else: |
| 253 | try: |
| 254 | await download_file(session, dst, file["links"]["self"]) |
| 255 | except: |
| 256 | continue |
| 257 | if (filetype == "pptx" and not ppt_validate(dst)) or ( |
| 258 | filetype == "pdf" and not pdf_validate(dst) |
| 259 | ): |
| 260 | shutil.rmtree(os.path.dirname(dst)) |
| 261 | continue |
| 262 | selected.append( |
| 263 | { |
| 264 | "filename": file["key"], |
| 265 | "size": file["size"], |
| 266 | "url": file["links"]["self"], |
| 267 | "license": license, |
| 268 | "title": record["title"], |
| 269 | "created": record["created"], |
| 270 | "updated": record["updated"], |
| 271 | "doi": record.get("doi", "unknown"), |
| 272 | "checksum": file["checksum"], |
| 273 | "page": page, |
| 274 | "topic": topic, |
| 275 | "filetype": filetype, |
| 276 | } |
no test coverage detected