| 413 | } |
| 414 | |
| 415 | void server_models::unload_lru() { |
| 416 | if (base_params.models_max <= 0) { |
| 417 | return; // no limit |
| 418 | } |
| 419 | // remove one of the servers if we passed the models_max (least recently used - LRU) |
| 420 | std::string lru_model_name = ""; |
| 421 | int64_t lru_last_used = ggml_time_ms(); |
| 422 | size_t count_active = 0; |
| 423 | { |
| 424 | std::unique_lock<std::mutex> lk(mutex); |
| 425 | for (const auto & m : mapping) { |
| 426 | if (m.second.meta.is_active()) { |
| 427 | count_active++; |
| 428 | if (m.second.meta.last_used < lru_last_used) { |
| 429 | lru_model_name = m.first; |
| 430 | lru_last_used = m.second.meta.last_used; |
| 431 | } |
| 432 | } |
| 433 | } |
| 434 | } |
| 435 | if (!lru_model_name.empty() && count_active >= (size_t)base_params.models_max) { |
| 436 | SRV_INF("models_max limit reached, removing LRU name=%s\n", lru_model_name.c_str()); |
| 437 | unload(lru_model_name); |
| 438 | // wait for unload to complete |
| 439 | { |
| 440 | std::unique_lock<std::mutex> lk(mutex); |
| 441 | cv.wait(lk, [this, &lru_model_name]() { |
| 442 | return mapping[lru_model_name].meta.status == SERVER_MODEL_STATUS_UNLOADED; |
| 443 | }); |
| 444 | } |
| 445 | } |
| 446 | } |
| 447 | |
| 448 | void server_models::load(const std::string & name) { |
| 449 | if (!has_model(name)) { |
nothing calls this directly
no test coverage detected