(task_batches)
| 297 | |
| 298 | # Create a generator that processes batches asynchronously |
| 299 | def async_process_batches(task_batches): |
| 300 | max_workers = 10 |
| 301 | with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: |
| 302 | futures = [executor.submit(process_batch, batch) for batch in task_batches] |
| 303 | for future in concurrent.futures.as_completed(futures): |
| 304 | yield future.result() |
| 305 | |
| 306 | def debug_print(msg): |
| 307 | if debug: |