| 214 | return len(prompt_ids) + image_tokens + img_count + audio_tokens + audio_count |
| 215 | |
| 216 | async def loop_for_request(self): |
| 217 | assert self.args.node_rank > 0 |
| 218 | while True: |
| 219 | ( |
| 220 | prompt, |
| 221 | sampling_params, |
| 222 | multimodal_params, |
| 223 | ) = await self.multinode_req_manager.recv_pyobj() |
| 224 | results_generator = self.generate(prompt, sampling_params, multimodal_params, None) |
| 225 | |
| 226 | async def generate_wrapper(results_generator): |
| 227 | async for _, _, _, _ in results_generator: |
| 228 | pass |
| 229 | |
| 230 | asyncio.create_task(generate_wrapper(results_generator)) |
| 231 | return |
| 232 | |
| 233 | def alloc_req_id(self, sampling_params, is_health_req: bool = False): |
| 234 | # 请求的 id 可以由外部传入，也可以由内部生成，但是由外部传入的时候，要自己保证全局唯一性 |