| 48 | |
| 49 | |
| 50 | def get_server(path_server: str, path_log: Optional[str]) -> dict: |
| 51 | if path_server.startswith("http://") or path_server.startswith("https://"): |
| 52 | return {"process": None, "address": path_server, "fout": None} |
| 53 | if os.environ.get("LLAMA_ARG_HOST") is None: |
| 54 | logger.info("LLAMA_ARG_HOST not explicitly set, using 127.0.0.1") |
| 55 | os.environ["LLAMA_ARG_HOST"] = "127.0.0.1" |
| 56 | if os.environ.get("LLAMA_ARG_PORT") is None: |
| 57 | logger.info("LLAMA_ARG_PORT not explicitly set, using 8080") |
| 58 | os.environ["LLAMA_ARG_PORT"] = "8080" |
| 59 | hostname: Optional[str] = os.environ.get("LLAMA_ARG_HOST") |
| 60 | port: Optional[str] = os.environ.get("LLAMA_ARG_PORT") |
| 61 | assert hostname is not None |
| 62 | assert port is not None |
| 63 | address: str = f"http://{hostname}:{port}" |
| 64 | logger.info(f"Starting the llama.cpp server under {address}...") |
| 65 | |
| 66 | fout = open(path_log.format(port=port), "w") if path_log is not None else subprocess.DEVNULL |
| 67 | process = subprocess.Popen([path_server], stdout=fout, stderr=subprocess.STDOUT) |
| 68 | |
| 69 | n_failures: int = 0 |
| 70 | while True: |
| 71 | try: |
| 72 | sleep(1.0) |
| 73 | exit_code = process.poll() |
| 74 | if exit_code is not None: |
| 75 | raise RuntimeError(f"llama.cpp server exited unexpectedly with exit code {exit_code}{path_log and f', see {path_log.format(port=port)}' or ''}") |
| 76 | response = requests.get(f"{address}/health") |
| 77 | if response.status_code == 200: |
| 78 | break |
| 79 | except requests.ConnectionError: |
| 80 | n_failures += 1 |
| 81 | if n_failures >= 10: |
| 82 | raise RuntimeError("llama.cpp server is not healthy after 10 seconds") |
| 83 | |
| 84 | return {"process": process, "address": address, "fout": fout} |
| 85 | |
| 86 | |
| 87 | def get_prompt_length(data: dict) -> int: |