Wait until the server accepts TCP connections, or raise if it dies/times out.
(
proc: subprocess.Popen,
host: str,
port: int,
timeout_s: float,
server_log: pathlib.Path | None = None,
)
| 620 | |
| 621 | |
| 622 | def wait_for_server_ready( |
| 623 | proc: subprocess.Popen, |
| 624 | host: str, |
| 625 | port: int, |
| 626 | timeout_s: float, |
| 627 | server_log: pathlib.Path | None = None, |
| 628 | ) -> None: |
| 629 | """Wait until the server accepts TCP connections, or raise if it dies/times out.""" |
| 630 | deadline = time.monotonic() + timeout_s |
| 631 | while True: |
| 632 | if proc.poll() is not None: |
| 633 | log_info = "" |
| 634 | if server_log is not None: |
| 635 | log_info = f"\nServer output:\n{_dump_server_log(server_log)}" |
| 636 | raise AssertionError( |
| 637 | f"Model server failed to start.\nreturncode={proc.returncode}{log_info}" |
| 638 | ) |
| 639 | try: |
| 640 | with socket.create_connection((host, port), timeout=1.0): |
| 641 | elapsed = time.monotonic() - deadline + timeout_s |
| 642 | print(f"Model server ready after {elapsed:.1f}s.") |
| 643 | return |
| 644 | except OSError: |
| 645 | if time.monotonic() >= deadline: |
| 646 | if proc.poll() is None: |
| 647 | proc.terminate() |
| 648 | try: |
| 649 | proc.wait(timeout=15) |
| 650 | except subprocess.TimeoutExpired: |
| 651 | proc.kill() |
| 652 | proc.wait(timeout=15) |
| 653 | log_info = "" |
| 654 | if server_log is not None: |
| 655 | log_info = f"\nServer output:\n{_dump_server_log(server_log)}" |
| 656 | raise AssertionError( |
| 657 | "Model server did not become ready before timeout.\n" |
| 658 | f"timeout_seconds={timeout_s}\n" |
| 659 | f"Set the corresponding env var to override.{log_info}" |
| 660 | ) |
| 661 | time.sleep(0.5) |
| 662 | |
| 663 | |
| 664 | def run_subprocess_step( |