()
| 87 | |
| 88 | |
| 89 | def main() -> int: |
| 90 | require_path(SERVER_BIN, "dflash_server", executable=True) |
| 91 | require_path(TARGET, "target GGUF") |
| 92 | require_path(DRAFT, "draft GGUF") |
| 93 | |
| 94 | LOG_PATH.parent.mkdir(parents=True, exist_ok=True) |
| 95 | log_f = LOG_PATH.open("w") |
| 96 | cmd = [ |
| 97 | str(SERVER_BIN), |
| 98 | str(TARGET), |
| 99 | "--draft", str(DRAFT), |
| 100 | "--max-ctx", "16384", |
| 101 | "--port", str(PORT), |
| 102 | "--prefix-cache-slots", "0", |
| 103 | "--prefill-cache-slots", "2", |
| 104 | "--ddtree", |
| 105 | "--ddtree-budget", "16", |
| 106 | "--cache-type-k", "tq3_0", |
| 107 | "--cache-type-v", "tq3_0", |
| 108 | "--fa-window", "0", |
| 109 | ] |
| 110 | proc = subprocess.Popen(cmd, stdout=log_f, stderr=subprocess.STDOUT, bufsize=1) |
| 111 | |
| 112 | def cleanup() -> None: |
| 113 | if proc.poll() is None: |
| 114 | proc.send_signal(signal.SIGINT) |
| 115 | try: |
| 116 | proc.wait(timeout=20) |
| 117 | except subprocess.TimeoutExpired: |
| 118 | proc.kill() |
| 119 | log_f.close() |
| 120 | |
| 121 | atexit.register(cleanup) |
| 122 | wait_server(proc) |
| 123 | |
| 124 | props = get_json("/props") |
| 125 | full_cache = props.get("full_cache", {}) |
| 126 | print(f"startup full_cache={full_cache}", flush=True) |
| 127 | if not full_cache.get("enabled") or full_cache.get("capacity") != 2: |
| 128 | raise RuntimeError(f"full cache not active in /props: {full_cache}") |
| 129 | |
| 130 | filler = ( |
| 131 | "The repository contains an inference server, a benchmark harness, " |
| 132 | "and cache implementations. This sentence is deterministic filler. " |
| 133 | ) |
| 134 | prompt = (filler * 240) + "\n\nQuestion: Reply with exactly the word cached." |
| 135 | payload = { |
| 136 | "model": "dflash", |
| 137 | "messages": [{"role": "user", "content": prompt}], |
| 138 | "max_tokens": 8, |
| 139 | "temperature": 0.0, |
| 140 | "stream": False, |
| 141 | } |
| 142 | |
| 143 | prefill = [] |
| 144 | for i in range(3): |
| 145 | t0 = time.time() |
| 146 | resp = post_json("/v1/chat/completions", payload) |
no test coverage detected