Function llama_reserve_model_kv_cache

llama.cpp:3392–3421

}

void llama_reserve_model_kv_cache(llama_model *model, const llama_context_params *cparams) {
#if defined(GGML_USE_CUBLAS)
    if (!ggml_cublas_loaded()) {
        throw std::runtime_error(format("cannot offload to GPU: " GGML_CUDA_NAME " not loaded"));
    }

    const llama_hparams &hparams = model->hparams;
    if (model->n_gpu_layers < hparams.n_layer + 1) {
        // should only reserve kv cache for models with all layers offloaded
        return;
    }

    const uint32_t n_embd  = hparams.n_embd_gqa();
    const uint32_t n_layer = hparams.n_layer;

    const int64_t n_mem      = n_layer*cparams->n_ctx;
    const int64_t n_elements = n_embd*n_mem;

    const ggml_type wtype = cparams->f16_kv ? GGML_TYPE_F16 : GGML_TYPE_F32;
    const size_t cache_size = n_elements*ggml_type_size(wtype);

    // reserve for k cache and v cache
    for (int i = 0; i < 2; i++) {
        if (!llama_reduce_vram_budget(cache_size)) {
            return;
        }
        model->n_gpu_layers++;
    }
#endif
}

static void llm_load_tensors(
        llama_model_loader & ml,