| 109 | |
| 110 | |
| 111 | class WorkItemProcessor: |
| 112 | def __init__(self): |
| 113 | self._pending = [] |
| 114 | self._permutations = {} |
| 115 | self.max_thread_count = multiprocessing.cpu_count() |
| 116 | self._thread_count = 0 |
| 117 | self._has_errors = False |
| 118 | self._process_done = threading.Event() |
| 119 | self._mutex = threading.Lock() |
| 120 | self._print_mutex = threading.Lock() |
| 121 | |
| 122 | def process(self, items): |
| 123 | self._pending = list(items) |
| 124 | self._thread_count = 0 |
| 125 | self._has_errors = False |
| 126 | self._process_done.clear() |
| 127 | |
| 128 | while self._pending: |
| 129 | item = self._next() |
| 130 | if item: |
| 131 | with self._mutex: |
| 132 | self._thread_count += item.permutations |
| 133 | threading.Thread(target=self._worker, args=(item,)).start() |
| 134 | else: |
| 135 | self._process_done.wait() |
| 136 | self._process_done.clear() |
| 137 | |
| 138 | logging.debug('Waiting for the last shaders to compile') |
| 139 | while True: |
| 140 | with self._mutex: |
| 141 | if self._thread_count <= 0: |
| 142 | break |
| 143 | self._process_done.wait() |
| 144 | self._process_done.clear() |
| 145 | |
| 146 | logging.debug('Done building all files') |
| 147 | return not self._has_errors |
| 148 | |
| 149 | def _next(self): |
| 150 | for i in range(len(self._pending)): |
| 151 | item = self._pending[i] |
| 152 | if item.permutations == 0: |
| 153 | if item.src not in self._permutations: |
| 154 | self._permutations[item.src] = item.get_permutations() |
| 155 | item.permutations = self._permutations[item.src] |
| 156 | |
| 157 | with self._mutex: |
| 158 | if self._thread_count == 0 or item.permutations + self._thread_count <= self.max_thread_count: |
| 159 | return self._pending.pop(i) |
| 160 | |
| 161 | return None |
| 162 | |
| 163 | def _worker(self, item): |
| 164 | logging.info(f'Building {item}') |
| 165 | try: |
| 166 | logging.debug(f'Spawning {" ".join(str(line) for line in item.get_command_line())}') |
| 167 | os.makedirs(Path(item.dest).parent, exist_ok=True) |
| 168 | p = subprocess.Popen(item.get_command_line(), stdout=subprocess.PIPE) |