start thread to get tokens
(self)
| 183 | return task.num_computed_tokens >= task.need_prefill_tokens |
| 184 | |
| 185 | def run(self): |
| 186 | """ |
| 187 | start thread to get tokens |
| 188 | """ |
| 189 | assert self.resource_manager is not None, "The resource manager is None, cannot run." |
| 190 | if self.worker is not None: |
| 191 | raise Exception("Worker is already running!") |
| 192 | |
| 193 | if envs.FD_USE_GET_SAVE_OUTPUT_V1: |
| 194 | self.worker = threading.Thread(target=self.process_sampling_results_use_zmq) |
| 195 | else: |
| 196 | self.worker = threading.Thread(target=self.process_sampling_results) |
| 197 | |
| 198 | self.worker.daemon = True |
| 199 | self.worker.start() |
| 200 | |
| 201 | def _reschedule_preempt_task_use_zmq(self, datas): |
| 202 | """reschedule when real batch size is smaller than the insert position of preemted_task""" |