Method remove_token_range

cactus/engine/engine_cache.cpp:478–512

}

void KVCache::remove_token_range(size_t start, size_t count) {
    if (count == 0 || start >= current_seq_len || start + count > current_seq_len) return;

    size_t tail_tokens = current_seq_len - start - count;

    auto erase_bytes = [](std::vector<uint8_t>& buf, size_t offset, size_t remove, size_t tail) {
        std::memmove(buf.data() + offset, buf.data() + offset + remove, tail);
        buf.resize(buf.size() - remove);
    };

    auto erase_floats = [](std::vector<float>& buf, size_t offset, size_t remove, size_t tail) {
        std::memmove(buf.data() + offset, buf.data() + offset + remove, tail * sizeof(float));
        buf.resize(buf.size() - remove);
    };

    for (size_t i = 0; i < layer_caches.size(); i++) {
        size_t dim = get_layer_head_dim(i);
        if (dim == 0) continue;
        auto& layer = layer_caches[i];
        size_t num_kv_heads = get_layer_kv_heads(i);
        size_t bytes_per_tok = num_kv_heads * dim * element_size;

        erase_bytes(layer.keys,   start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok);
        erase_bytes(layer.values, start * bytes_per_tok, count * bytes_per_tok, tail_tokens * bytes_per_tok);

        if (precision == Precision::INT8 && !layer.key_scales.empty()) {
            size_t scaler_per_tok = num_kv_heads * dim / KV_QUANT_GROUP_SIZE;
            erase_floats(layer.key_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok);
            erase_floats(layer.value_scales, start * scaler_per_tok, count * scaler_per_tok, tail_tokens * scaler_per_tok);
        }
    }

    current_seq_len -= count;
    total_seq_len -= count;
}

void KVCache::compact_to_windows(const std::vector<size_t>& target_windows) {
    for (size_t i = 0; i < layer_caches.size(); i++) {