| 296 | |
| 297 | |
| 298 | def check_files_md5_parallel(directory: str, num_workers: int = 8): |
| 299 | filepaths = [] |
| 300 | for root, _, files in os.walk(directory): |
| 301 | for file in files: |
| 302 | if file.endswith(".pptx") or file.endswith(".pdf"): |
| 303 | filepaths.append(os.path.join(root, file)) |
| 304 | with ProcessPoolExecutor(max_workers=num_workers) as executor: |
| 305 | result = list(executor.map(verify_md5, filepaths)) |
| 306 | with jsonlines.open("incorrect_files.jsonl", mode="w") as writer: |
| 307 | for i in result: |
| 308 | if i is not None: |
| 309 | writer.write(i) |
| 310 | |
| 311 | |
| 312 | def dataset_stat(): |