(endpoint: str, inference_parallel_size: int, uds: str | None = None)
| 31 | |
| 32 | |
| 33 | def check_vllm_ready(endpoint: str, inference_parallel_size: int, uds: str | None = None): |
| 34 | if rank != rank // inference_parallel_size * inference_parallel_size: |
| 35 | return |
| 36 | retry_num = 0 |
| 37 | transport = None |
| 38 | if uds is not None: |
| 39 | transport = httpx.HTTPTransport(uds=uds) |
| 40 | while True: |
| 41 | try: |
| 42 | response = httpx.Client(transport=transport).get(f"{endpoint}/health", timeout=10) |
| 43 | response.raise_for_status() |
| 44 | break |
| 45 | except (httpx.ConnectError, httpx.HTTPStatusError) as e: |
| 46 | retry_num += 1 |
| 47 | logger.warning(f"fail to check vllm ready, retry {retry_num} times, error: {e}") |
| 48 | time.sleep(5) |
| 49 | |
| 50 | |
| 51 | def split_checkpoint_files(checkpoint_path: str, rank: int, world_size: int) -> list[str]: |
no outgoing calls
no test coverage detected