| 43 | |
| 44 | @wraps(func) |
| 45 | def wrapper(iterable, *args, **kwargs): |
| 46 | results = [] |
| 47 | # Create a tqdm progress bar with the total number of items to process |
| 48 | with tqdm( |
| 49 | unit_scale=True, |
| 50 | unit_divisor=1024, |
| 51 | initial=0, |
| 52 | total=len(iterable), |
| 53 | desc=tqdm_desc or f'Processing {len(iterable)} items', |
| 54 | disable=disable_tqdm, |
| 55 | ) as pbar: |
| 56 | # Define a wrapper function to update the progress bar |
| 57 | with ThreadPoolExecutor(max_workers=max_workers) as executor: |
| 58 | # Submit all tasks |
| 59 | futures = { |
| 60 | executor.submit(func, item, *args, **kwargs): item |
| 61 | for item in iterable |
| 62 | } |
| 63 | |
| 64 | # Update the progress bar as tasks complete |
| 65 | failed_items = [] |
| 66 | for future in as_completed(futures): |
| 67 | pbar.update(1) |
| 68 | if fault_tolerant: |
| 69 | try: |
| 70 | results.append(future.result()) |
| 71 | except Exception as e: |
| 72 | item = futures[future] |
| 73 | logger.error(f'Task failed for {item}: {e}') |
| 74 | failed_items.append((item, e)) |
| 75 | else: |
| 76 | results.append(future.result()) |
| 77 | if fault_tolerant: |
| 78 | return results, failed_items |
| 79 | return results |
| 80 | |
| 81 | return wrapper |
| 82 | |