MCPcopy Create free account
hub / github.com/google-research/language / __init__

Method __init__

language/realm/parallel.py:78–106  ·  view source on GitHub ↗
(self,
               create_worker,
               queue_size,
               num_workers=None,
               worker_kwargs=None)

Source from the content-addressed store, hash-verified

76 """
77
78 def __init__(self,
79 create_worker,
80 queue_size,
81 num_workers=None,
82 worker_kwargs=None):
83 if num_workers is None:
84 # Use all CPUs.
85 num_workers = os.cpu_count()
86
87 if worker_kwargs is None:
88 worker_kwargs = {}
89
90 self._mp = multiprocessing
91
92 self._input_queue = self._mp.Queue(queue_size)
93 self._output_queue = self._mp.Queue(queue_size)
94 self._worker_processes = []
95 self._submitter_processes = []
96 self._stop_event = self._mp.Event()
97
98 for i in range(num_workers):
99 worker = create_worker(**worker_kwargs)
100 process = self._mp.Process( # pytype: disable=attribute-error # re-none
101 target=process_from_queue,
102 name='worker-{}'.format(i),
103 args=(self._input_queue, self._output_queue, worker,
104 self._stop_event))
105 process.start()
106 self._worker_processes.append(process)
107
108 def _shutdown_queue(self, q):
109 """Shuts down the multiprocessing Queue."""

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected