| 1417 | } |
| 1418 | |
| 1419 | bool update_slots() { |
| 1420 | // attend tasks |
| 1421 | process_tasks(); |
| 1422 | |
| 1423 | // update the system prompt wait until all slots are idle state |
| 1424 | if (system_need_update && all_slots_are_idle) |
| 1425 | { |
| 1426 | LOG_TEE("updating system prompt\n"); |
| 1427 | update_system_prompt(); |
| 1428 | } |
| 1429 | |
| 1430 | llama_batch_clear(batch); |
| 1431 | |
| 1432 | if (all_slots_are_idle) |
| 1433 | { |
| 1434 | if (system_prompt.empty() && clean_kv_cache) |
| 1435 | { |
| 1436 | LOG_TEE("all slots are idle and system prompt is empty, clear the KV cache\n"); |
| 1437 | kv_cache_clear(); |
| 1438 | } |
| 1439 | // avoid 100% usage of cpu all time |
| 1440 | std::this_thread::sleep_for(std::chrono::milliseconds(5)); |
| 1441 | } |
| 1442 | |
| 1443 | for (llama_client_slot &slot : slots) |
| 1444 | { |
| 1445 | if (slot.is_processing() && slot.cache_tokens.size() >= (size_t) slot.n_ctx) |
| 1446 | { |
| 1447 | // Shift context |
| 1448 | const int n_left = slot.n_past - slot.params.n_keep - 1; |
| 1449 | const int n_discard = n_left / 2; |
| 1450 | |
| 1451 | LOG_TEE("slot %d: context shift - n_keep = %d, n_left = %d, n_discard = %d\n", slot.id, slot.params.n_keep, n_left, n_discard); |
| 1452 | llama_kv_cache_seq_rm (ctx, slot.id, slot.params.n_keep + 1 , slot.params.n_keep + n_discard + 1); |
| 1453 | llama_kv_cache_seq_shift(ctx, slot.id, slot.params.n_keep + 1 + n_discard, slot.n_past, -n_discard); |
| 1454 | |
| 1455 | for (size_t i = slot.params.n_keep + 1 + n_discard; i < slot.cache_tokens.size(); i++) |
| 1456 | { |
| 1457 | slot.cache_tokens[i - n_discard] = slot.cache_tokens[i]; |
| 1458 | } |
| 1459 | |
| 1460 | slot.cache_tokens.resize(slot.cache_tokens.size() - n_discard); |
| 1461 | |
| 1462 | slot.n_past -= n_discard; |
| 1463 | |
| 1464 | slot.truncated = true; |
| 1465 | |
| 1466 | LOG_VERBOSE("context shift", { |
| 1467 | {"n_ctx", n_ctx}, |
| 1468 | {"n_keep", params.n_keep}, |
| 1469 | {"n_left", n_left}, |
| 1470 | }); |
| 1471 | } |
| 1472 | } |
| 1473 | |
| 1474 | // decode any currently ongoing sequences |
| 1475 | for (auto & slot : slots) |
| 1476 | { |
no test coverage detected