r"""Starts the vllm server in a subprocess.
(self)
| 79 | ) |
| 80 | |
| 81 | def _start_server(self) -> None: |
| 82 | r"""Starts the vllm server in a subprocess.""" |
| 83 | try: |
| 84 | subprocess.Popen( |
| 85 | ["vllm", "server", "--port", "8000"], |
| 86 | stdout=subprocess.PIPE, |
| 87 | stderr=subprocess.PIPE, |
| 88 | ) |
| 89 | self._url = "http://localhost:8000/v1" |
| 90 | print( |
| 91 | f"vllm server started on {self._url} " |
| 92 | f"for {self.model_type} model." |
| 93 | ) |
| 94 | except Exception as e: |
| 95 | print(f"Failed to start vllm server: {e}.") |
| 96 | |
| 97 | @property |
| 98 | def token_counter(self) -> BaseTokenCounter: |