Async add request Args: request_id: Request ID prompt: Input prompt sampling_params: Sampling parameters arrival_time: Arrival time **kwargs: Other parameters
(
self,
request_id: str,
prompt: Union[str, List[str], Dict[str, Any]],
sampling_params: Optional[SamplingParams] = None,
arrival_time: Optional[float] = None,
**kwargs,
)
| 354 | ) |
| 355 | |
| 356 | async def add_request( |
| 357 | self, |
| 358 | request_id: str, |
| 359 | prompt: Union[str, List[str], Dict[str, Any]], |
| 360 | sampling_params: Optional[SamplingParams] = None, |
| 361 | arrival_time: Optional[float] = None, |
| 362 | **kwargs, |
| 363 | ): |
| 364 | """ |
| 365 | Async add request |
| 366 | |
| 367 | Args: |
| 368 | request_id: Request ID |
| 369 | prompt: Input prompt |
| 370 | sampling_params: Sampling parameters |
| 371 | arrival_time: Arrival time |
| 372 | **kwargs: Other parameters |
| 373 | |
| 374 | """ |
| 375 | |
| 376 | if request_id is None: |
| 377 | request_id = str(uuid.uuid4()) |
| 378 | |
| 379 | if arrival_time is None: |
| 380 | arrival_time = time.time() |
| 381 | |
| 382 | if isinstance(prompt, str): |
| 383 | prompt = { |
| 384 | "prompt": prompt, |
| 385 | "request_id": request_id, |
| 386 | } |
| 387 | elif isinstance(prompt, list) and isinstance(prompt[0], int): |
| 388 | prompt = { |
| 389 | "prompt_token_ids": prompt, |
| 390 | "request_id": request_id, |
| 391 | } |
| 392 | elif isinstance(prompt, dict): |
| 393 | prompt["request_id"] = request_id |
| 394 | else: |
| 395 | raise TypeError(f"Invalid type for 'prompt': {type(prompt)}, expected one of ['str', 'list', 'dict'].") |
| 396 | |
| 397 | if sampling_params is not None: |
| 398 | prompt.update(asdict(sampling_params)) |
| 399 | prompt["metrics"] = {} |
| 400 | |
| 401 | try: |
| 402 | # Check if already preprocessed by api_server |
| 403 | is_preprocessed = prompt.get("_preprocessed", False) |
| 404 | |
| 405 | if inspect.iscoroutinefunction(self.data_processor.process_request_dict): |
| 406 | request = await self.data_processor.process_request_dict(prompt, self.cfg.model_config.max_model_len) |
| 407 | else: |
| 408 | request = self.data_processor.process_request_dict(prompt, self.cfg.model_config.max_model_len) |
| 409 | |
| 410 | request["prompt_token_ids_len"] = len(request["prompt_token_ids"]) |
| 411 | |
| 412 | # Cache prompt metadata for later enrichment of async responses |
| 413 | req_id = request.get("request_id") |
no test coverage detected