Generate key to group compatible requests Args: request_data: The request data dictionary Returns: String key for grouping compatible requests
(self, request_data: Dict[str, Any])
| 80 | f"max_wait_ms={max_wait_ms}") |
| 81 | |
| 82 | def _get_request_key(self, request_data: Dict[str, Any]) -> str: |
| 83 | """ |
| 84 | Generate key to group compatible requests |
| 85 | |
| 86 | Args: |
| 87 | request_data: The request data dictionary |
| 88 | |
| 89 | Returns: |
| 90 | String key for grouping compatible requests |
| 91 | """ |
| 92 | model = request_data.get('model', 'default') |
| 93 | approach = request_data.get('optillm_approach', 'none') |
| 94 | |
| 95 | # Stream requests cannot be batched |
| 96 | if request_data.get('stream', False): |
| 97 | raise BatchingError("Streaming requests cannot be batched") |
| 98 | |
| 99 | return f"{model}:{approach}" |
| 100 | |
| 101 | def _validate_batch_compatibility(self, requests: List[BatchRequest]) -> None: |
| 102 | """ |
no test coverage detected