MCPcopy
hub / github.com/omkarcloud/botasaurus / run_parallel

Function run_parallel

botasaurus/decorators_common.py:105–127  ·  view source on GitHub ↗
(run, ls, n_workers, use_threads)

Source from the content-addressed store, hash-verified

103 return flatten(self.result_list)
104
105def run_parallel(run, ls, n_workers, use_threads):
106 from joblib import Parallel, delayed
107 from .thread_with_result import ThreadWithResult
108 import sys
109
110 if use_threads:
111 execute_parallel_tasks = lambda: Parallel(n_jobs=n_workers, backend="threading")(
112 delayed(run)(l) for l in ls
113 )
114 else:
115 execute_parallel_tasks = lambda: Parallel(n_jobs=n_workers)(
116 delayed(run)(l) for l in ls
117 )
118
119 parallel_thread = ThreadWithResult(target=execute_parallel_tasks, daemon=True)
120 parallel_thread.start()
121 try:
122 while parallel_thread.is_alive():
123 parallel_thread.join(0.2) # time out not to block KeyboardInterrupt
124 except KeyboardInterrupt:
125 sys.exit(1)
126
127 return parallel_thread.result
128
129
130class AsyncResult:

Callers 3

wrapper_requestsFunction · 0.90
wrapper_browserFunction · 0.90
wrapper_requestsFunction · 0.90

Calls 3

joinMethod · 0.95
ThreadWithResultClass · 0.85
startMethod · 0.45

Tested by

no test coverage detected