(
self,
request_id: str,
prompt: Optional[str],
sampling_params: SamplingParams,
prompt_token_ids: Optional[List[int]] = None,
arrival_time: Optional[float] = None,
)
| 361 | await asyncio.sleep(0) |
| 362 | |
| 363 | async def add_request( |
| 364 | self, |
| 365 | request_id: str, |
| 366 | prompt: Optional[str], |
| 367 | sampling_params: SamplingParams, |
| 368 | prompt_token_ids: Optional[List[int]] = None, |
| 369 | arrival_time: Optional[float] = None, |
| 370 | ) -> AsyncStream: |
| 371 | if self.log_requests: |
| 372 | shortened_prompt = prompt |
| 373 | shortened_token_ids = prompt_token_ids |
| 374 | if self.max_log_len is not None: |
| 375 | if shortened_prompt is not None: |
| 376 | shortened_prompt = shortened_prompt[:self.max_log_len] |
| 377 | if shortened_token_ids is not None: |
| 378 | shortened_token_ids = shortened_token_ids[:self. |
| 379 | max_log_len] |
| 380 | logger.info(f"Received request {request_id}: " |
| 381 | f"prompt: {shortened_prompt!r}, " |
| 382 | f"sampling params: {sampling_params}, " |
| 383 | f"prompt token ids: {shortened_token_ids}.") |
| 384 | |
| 385 | if not self.is_running: |
| 386 | if self.start_engine_loop: |
| 387 | self.start_background_loop() |
| 388 | else: |
| 389 | raise AsyncEngineDeadError( |
| 390 | "Background loop is not running. If it was running, " |
| 391 | "inspect the output to find the stacktrace of the " |
| 392 | "error that caused the background loop to stop " |
| 393 | "(AsyncEngineDeadError).") |
| 394 | |
| 395 | stream = self._request_tracker.add_request( |
| 396 | request_id, |
| 397 | prompt=prompt, |
| 398 | sampling_params=sampling_params, |
| 399 | prompt_token_ids=prompt_token_ids, |
| 400 | arrival_time=arrival_time) |
| 401 | |
| 402 | return stream |
| 403 | |
| 404 | async def generate( |
| 405 | self, |
no test coverage detected