(
self,
model_type: Union[ModelType, str],
model_config_dict: Optional[Dict[str, Any]] = None,
api_key: Optional[str] = None,
url: Optional[str] = None,
token_counter: Optional[BaseTokenCounter] = None,
)
| 55 | """ |
| 56 | |
| 57 | def __init__( |
| 58 | self, |
| 59 | model_type: Union[ModelType, str], |
| 60 | model_config_dict: Optional[Dict[str, Any]] = None, |
| 61 | api_key: Optional[str] = None, |
| 62 | url: Optional[str] = None, |
| 63 | token_counter: Optional[BaseTokenCounter] = None, |
| 64 | ) -> None: |
| 65 | if model_config_dict is None: |
| 66 | model_config_dict = VLLMConfig().as_dict() |
| 67 | url = url or os.environ.get("VLLM_BASE_URL") |
| 68 | super().__init__( |
| 69 | model_type, model_config_dict, api_key, url, token_counter |
| 70 | ) |
| 71 | if not self._url: |
| 72 | self._start_server() |
| 73 | # Use OpenAI cilent as interface call vLLM |
| 74 | self._client = OpenAI( |
| 75 | timeout=500, |
| 76 | max_retries=3, |
| 77 | api_key="EMPTY", # required but ignored |
| 78 | base_url=self._url, |
| 79 | ) |
| 80 | |
| 81 | def _start_server(self) -> None: |
| 82 | r"""Starts the vllm server in a subprocess.""" |
nothing calls this directly
no test coverage detected