(file_type: str, func: callable, thread_num: int, topic="*")
| 48 | |
| 49 | |
| 50 | def process_filetype(file_type: str, func: callable, thread_num: int, topic="*"): |
| 51 | folders = glob.glob(f"data/{topic}/{file_type}/*") |
| 52 | progress_bar = tqdm(total=len(folders), desc=f"processing {file_type}") |
| 53 | |
| 54 | def process_folder(folder, *args, **kwargs): |
| 55 | try: |
| 56 | func(folder, *args, **kwargs) |
| 57 | except Exception as e: |
| 58 | print(f"process {file_type} folder {folder} failed: {e}") |
| 59 | traceback.print_exc() |
| 60 | finally: |
| 61 | progress_bar.update(1) |
| 62 | |
| 63 | with ThreadPoolExecutor(thread_num) as executor: |
| 64 | list(executor.map(process_folder, folders, range(len(folders)))) |
| 65 | |
| 66 | progress_bar.close() |
| 67 | |
| 68 | |
| 69 | def parse_pdfs(pdf_folders: list[str], idx: int): |
no test coverage detected