| 69 | console.info("Setting up LLM...") |
| 70 | if local_agent != "None": |
| 71 | def run_cmd(): |
| 72 | with open(os.devnull, 'w') as devnull: |
| 73 | env = os.environ.copy() |
| 74 | env["CUDA_VISIBLE_DEVICES"] = devices |
| 75 | subprocess.run(["python", "-m", |
| 76 | "vllm.entrypoints.openai.api_server", |
| 77 | "--model", str(DEFAULT_LOCAL_AGENT), |
| 78 | "--trust-remote-code", |
| 79 | "--tensor-parallel-size", str(len(devices.split(","))), |
| 80 | "--port", str(DEFAULT_VLLM_PORT)], |
| 81 | stdout=devnull, stderr=devnull, env=env) |
| 82 | thread = threading.Thread(target=run_cmd) |
| 83 | thread.daemon = True |
| 84 | thread.start() |