Method handle_openai_chat_completion

src/server/rest_handler.cpp:1044–1244

@brief Handle the openai chat completion request @param request the request @param send_response the send response @param send_streaming_response the send streaming response

///@param send_response the send response
///@param send_streaming_response the send streaming response
void RestHandler::handle_openai_chat_completion(const json& request,
                                               std::function<void(const json&)> send_response,
                                               StreamResponseCallback send_streaming_response,
                                               std::shared_ptr<CancellationToken> cancellation_token) {
    static std::string model_used_for_last_message = "model-faker";
    try {
        // Extract OpenAI-style parameters
        json current_messages = request["messages"];
        std::string model = request.value("model", current_model_tag);
        bool stream = request.value("stream", false);
        int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096));
        json tools = request.value("tools", json::array());
        json options = request.value("options", json::object());

        auto load_start_time = time_utils::now();
        if (!ensure_model_loaded(model)) {
            json error_response = {{"error", "Failed to load " + model + " model!"}};
            send_response(error_response);
            return;
        }
        auto load_end_time = time_utils::now();

        configure_chat_engine_parameters(options, request);

        current_messages = normalize_messages(current_messages);
        current_messages = normalize_template(current_messages);

        // see if we can use prompt cache
        chat_meta_info_t meta_info;
        bool can_use_prompt_cache = false;
        if (model != model_used_for_last_message) { // switch models will clear context
            this->prompt_cache.update_message_checksum(current_messages);
            this->prompt_cache.update_tool_checksum(tools);
            model_used_for_last_message = model;
        }
        else {
            can_use_prompt_cache = prompt_cache.can_use_cache(current_messages, auto_chat_engine->get_chat_template_type(), tools);
            if (can_use_prompt_cache) {
                meta_info.restore_allowed = true;
                header_print("FLM", "Use cached prompt!");
            }
            else {
                // cannot use cache, clear and re-insert all
                auto_chat_engine->clear_context();
            }
        }

        if (model.starts_with("gemma4-it")) {
            current_messages = convert_tool_responses_gemma4(current_messages);
        }

        // std::cout << "FLM current_messages: \n" << current_messages.dump(4) << std::endl;

        lm_uniform_input_t uniformed_input;
        uniformed_input.messages = current_messages;
        uniformed_input.tools = tools;
        meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first;
        meta_info.max_prefill_len = this->prefill_chunk_len;