Function infer

core/inference.py:96–177

(messages: list[dict], stream: bool = True, extra_stop: list = None)

        _server_proc = None

def infer(messages: list[dict], stream: bool = True, extra_stop: list = None) -> str:
    global last_tps
    _start_server()
    cfg = MODEL_CONFIG

    stop_tokens = list(cfg["stop"]) + [
        "</tool>", "</write_file>", "</shell>",
        # Prevent model from echoing system prompt sections into its response
        "\n## Current Project", "\n## Project Map", "\n## Loaded Files",
        "\n## Project Memory", "\nuser\n", "\nUSER\n", "\nUser\n",
        "<|im_start|>user", "<|im_start|>system",
    ]
    if extra_stop:
        stop_tokens += [s for s in extra_stop if s not in stop_tokens]

    payload = json.dumps({
        "model":          "codey",
        "messages":       messages,
        "max_tokens":     cfg["max_tokens"],
        "temperature":    cfg["temperature"],
        "top_p":          cfg["top_p"],
        "top_k":          cfg["top_k"],
        "repeat_penalty": cfg["repeat_penalty"],
        "stop":           stop_tokens,
        "stream":         stream,
    }).encode("utf-8")

    req = urllib.request.Request(
        CHAT_URL,
        data=payload,
        headers={"Content-Type": "application/json"},
        method="POST",
    )

    response_text = ""

    _t0 = time.time()
    try:
        with urllib.request.urlopen(req, timeout=300) as resp:
            if stream:
                print("\033[1;32mCodey:\033[0m ", end="", flush=True)
                for raw_line in resp:
                    line = raw_line.decode("utf-8").strip()
                    if not line or not line.startswith("data: "):
                        continue
                    chunk = line[6:]
                    if chunk == "[DONE]":
                        break
                    try:
                        data = json.loads(chunk)
                        token = data["choices"][0]["delta"].get("content", "")
                        response_text += token
                        print(token, end="", flush=True)
                    except Exception:
                        continue
                print()
                _elapsed = time.time() - _t0
                if _elapsed > 0 and response_text: