()
| 14 | sys.exit(1) |
| 15 | |
| 16 | def run_server(): |
| 17 | build_dir = "build" |
| 18 | if platform.system() == "Windows": |
| 19 | server_path = os.path.join(build_dir, "bin", "Release", "llama-server.exe") |
| 20 | if not os.path.exists(server_path): |
| 21 | server_path = os.path.join(build_dir, "bin", "llama-server") |
| 22 | else: |
| 23 | server_path = os.path.join(build_dir, "bin", "llama-server") |
| 24 | |
| 25 | command = [ |
| 26 | f'{server_path}', |
| 27 | '-m', args.model, |
| 28 | '-c', str(args.ctx_size), |
| 29 | '-t', str(args.threads), |
| 30 | '-n', str(args.n_predict), |
| 31 | '-ngl', '0', |
| 32 | '--temp', str(args.temperature), |
| 33 | '--host', args.host, |
| 34 | '--port', str(args.port), |
| 35 | '-cb' # Enable continuous batching |
| 36 | ] |
| 37 | |
| 38 | if args.prompt: |
| 39 | command.extend(['-p', args.prompt]) |
| 40 | |
| 41 | # Note: -cnv flag is removed as it's not supported by the server |
| 42 | |
| 43 | print(f"Starting server on {args.host}:{args.port}") |
| 44 | run_command(command) |
| 45 | |
| 46 | def signal_handler(sig, frame): |
| 47 | print("Ctrl+C pressed, shutting down server...") |
no test coverage detected