Start llama-server subprocess.
(self)
| 42 | self._started = False |
| 43 | |
| 44 | def start(self) -> bool: |
| 45 | """Start llama-server subprocess.""" |
| 46 | try: |
| 47 | if self.process and self.process.poll() is None: |
| 48 | # Already running |
| 49 | return True |
| 50 | |
| 51 | # Check if llama-server is already running on port 8080 (e.g., from daemon) |
| 52 | if self._is_port_in_use(): |
| 53 | info(f"llama-server already running on port {self.port}, using existing server") |
| 54 | self._started = True |
| 55 | return True |
| 56 | |
| 57 | info(f"Starting llama-server...") |
| 58 | |
| 59 | # Build command |
| 60 | cmd = [ |
| 61 | str(LLAMA_SERVER_BIN), |
| 62 | "-m", str(self.model_path), |
| 63 | "--host", SERVER_HOST, |
| 64 | "--port", str(self.port), |
| 65 | "-c", str(MODEL_CONFIG["n_ctx"]), |
| 66 | "-t", str(MODEL_CONFIG["n_threads"]), |
| 67 | "--temp", str(MODEL_CONFIG["temperature"]), |
| 68 | "--top-p", str(MODEL_CONFIG["top_p"]), |
| 69 | "--top-k", str(MODEL_CONFIG["top_k"]), |
| 70 | "--repeat-penalty", str(MODEL_CONFIG["repeat_penalty"]), |
| 71 | "--n-predict", str(MODEL_CONFIG["max_tokens"]), |
| 72 | "--flash-attn", "on", # fused attention kernel, faster prefill |
| 73 | "--embedding", # enable /v1/embeddings endpoint for hybrid KB search |
| 74 | "--pooling", "mean", # mean pooling → single vector per input (OAI-compatible) |
| 75 | ] |
| 76 | |
| 77 | # Add stop tokens (using --reverse-prompt) |
| 78 | for stop in MODEL_CONFIG.get("stop", []): |
| 79 | cmd.extend(["--reverse-prompt", stop]) |
| 80 | |
| 81 | # ── mmap / mlock settings for the 7B model (Change 2) ────────── |
| 82 | # Pass --mmap / --no-mmap explicitly in both directions so the flag |
| 83 | # is visible in ps output and not left to llama.cpp's default. |
| 84 | # --no-mlock does NOT exist in this llama.cpp build; omitting --mlock |
| 85 | # is sufficient to keep mlock disabled (the llama.cpp default). |
| 86 | try: |
| 87 | from utils.config import QWEN_7B_MMAP, QWEN_7B_MLOCK |
| 88 | if QWEN_7B_MMAP: |
| 89 | cmd.append("--mmap") |
| 90 | else: |
| 91 | cmd.append("--no-mmap") |
| 92 | if QWEN_7B_MLOCK: |
| 93 | cmd.append("--mlock") |
| 94 | info( |
| 95 | f"7B model: mmap={'enabled' if QWEN_7B_MMAP else 'disabled'}, " |
| 96 | f"mlock={'enabled' if QWEN_7B_MLOCK else 'disabled'}" |
| 97 | ) |
| 98 | except ImportError: |
| 99 | pass # Config not available — use llama.cpp defaults (mmap on, mlock off) |
| 100 | |
| 101 | # Start process - redirect output to log file to avoid pipe buffer issues |
no test coverage detected