Manages a dedicated llama-server subprocess for embeddings only. Uses --embedding --pooling mean to expose /v1/embeddings in OAI format. nomic-embed-text-v1.5: 2048 ctx, 768-dim, 4 threads.
| 38 | |
| 39 | |
| 40 | class EmbedServer: |
| 41 | """ |
| 42 | Manages a dedicated llama-server subprocess for embeddings only. |
| 43 | |
| 44 | Uses --embedding --pooling mean to expose /v1/embeddings in OAI format. |
| 45 | nomic-embed-text-v1.5: 2048 ctx, 768-dim, 4 threads. |
| 46 | """ |
| 47 | |
| 48 | def __init__(self): |
| 49 | self.model_path = EMBED_MODEL_PATH |
| 50 | self.port = EMBED_SERVER_PORT |
| 51 | self.process: Optional[subprocess.Popen] = None |
| 52 | self._started = False |
| 53 | |
| 54 | # ── Lifecycle ────────────────────────────────────────────────────────────── |
| 55 | |
| 56 | def start(self) -> bool: |
| 57 | """Start the embed server subprocess. Idempotent.""" |
| 58 | # Already running as our own subprocess? |
| 59 | if self.process and self.process.poll() is None and self._check_health(): |
| 60 | return True |
| 61 | |
| 62 | # Kill any stale llama-server occupying the embed port — it may have |
| 63 | # different settings (wrong ctx, old ubatch) from a previous run. |
| 64 | if self._is_port_open(): |
| 65 | info(f"Stale process on port {self.port} — replacing with fresh embed server...") |
| 66 | self._kill_port_occupant() |
| 67 | |
| 68 | if not self.model_path.exists(): |
| 69 | warning(f"Embed model not found: {self.model_path}") |
| 70 | warning("Run: bash tools/setup_skills.sh to set up the embedding model") |
| 71 | return False |
| 72 | |
| 73 | llama_bin = Path(LLAMA_SERVER_BIN) |
| 74 | if not llama_bin.exists(): |
| 75 | error(f"llama-server binary not found: {LLAMA_SERVER_BIN}") |
| 76 | return False |
| 77 | |
| 78 | info(f"Starting embed server (nomic) on port {self.port}...") |
| 79 | |
| 80 | cmd = [ |
| 81 | str(llama_bin), |
| 82 | "-m", str(self.model_path), |
| 83 | "--host", _HOST, |
| 84 | "--port", str(self.port), |
| 85 | "-c", "2048", # 2k ctx — fast for 92% of chunks; rest use BM25 fallback |
| 86 | "-t", "2", # 2 threads for embedding (keep CPU headroom for 7B) |
| 87 | "-b", "2048", # logical batch size matches ctx |
| 88 | "--ubatch-size", "2048", # physical batch matches ctx |
| 89 | "--embedding", # enable /v1/embeddings endpoint |
| 90 | "--pooling", "mean",# OAI-compatible single vector per input |
| 91 | ] |
| 92 | |
| 93 | log_file = Path.home() / ".codey-v2" / "embed-server.log" |
| 94 | log_file.parent.mkdir(parents=True, exist_ok=True) |
| 95 | |
| 96 | log_fd = open(log_file, "a") |
| 97 | log_fd.write(f"\n--- embed server start: {' '.join(cmd)}\n") |