(self)
| 87 | ) |
| 88 | |
| 89 | def _start_server(self) -> None: |
| 90 | from sglang.utils import ( # type: ignore[import-untyped] |
| 91 | execute_shell_command, |
| 92 | wait_for_server, |
| 93 | ) |
| 94 | |
| 95 | try: |
| 96 | if not self._url: |
| 97 | cmd = ( |
| 98 | f"python -m sglang.launch_server " |
| 99 | f"--model-path {self.model_type} " |
| 100 | f"--port 30000 " |
| 101 | f"--host 0.0.0.0" |
| 102 | ) |
| 103 | |
| 104 | server_process = execute_shell_command(cmd) |
| 105 | wait_for_server("http://localhost:30000") |
| 106 | self._url = "http://127.0.0.1:30000/v1" |
| 107 | self.server_process = server_process |
| 108 | # Start the inactivity monitor in a background thread |
| 109 | self._inactivity_thread = threading.Thread( |
| 110 | target=self._monitor_inactivity, daemon=True |
| 111 | ) |
| 112 | self._inactivity_thread.start() |
| 113 | self.last_run_time = time.time() |
| 114 | # Initialize the client after the server starts |
| 115 | self._client = OpenAI( |
| 116 | timeout=180, |
| 117 | max_retries=3, |
| 118 | api_key="Set-but-ignored", # required but ignored |
| 119 | base_url=self._url, |
| 120 | ) |
| 121 | except Exception as e: |
| 122 | raise RuntimeError(f"Failed to start SGLang server: {e}") from e |
| 123 | |
| 124 | def _ensure_server_running(self) -> None: |
| 125 | r"""Ensures that the server is running. If not, starts the server.""" |
no test coverage detected