| 186 | } |
| 187 | |
| 188 | static size_t llama_set_vram_budget(double budget_gb, int gpu_device) { |
| 189 | #if defined(GGML_USE_CUBLAS) |
| 190 | if (!ggml_cublas_loaded()) { |
| 191 | throw std::runtime_error("CUDA is not loaded"); |
| 192 | } |
| 193 | |
| 194 | if (budget_gb < 0) { |
| 195 | // if the user didn't specify a budget, use all available memory |
| 196 | // and leave 256 MB as a safety margin |
| 197 | vram_budget_bytes = ggml_cuda_get_free_memory(gpu_device) - 256 * 1024 * 1024; |
| 198 | } else { |
| 199 | // otherwise, use the specified budget |
| 200 | vram_budget_bytes = (size_t) (budget_gb * 1024 * 1024 * 1024); |
| 201 | } |
| 202 | |
| 203 | return vram_budget_bytes; |
| 204 | #else |
| 205 | return 0; |
| 206 | #endif |
| 207 | } |
| 208 | |
| 209 | static bool llama_reduce_vram_budget(size_t budget_bytes) { |
| 210 | #if not defined(GGML_USE_CUBLAS) |