Method update_from_npu

cactus/engine/engine_cache.cpp:351–476

}

void KVCache::update_from_npu(size_t layer_idx, const __fp16* k_data, const __fp16* v_data,
                               size_t num_tokens, size_t kv_heads, size_t dim) {
    if (layer_idx >= num_layers || !k_data || !v_data || num_tokens == 0) {
        return;
    }

    auto& cache = layer_caches[layer_idx];
    size_t old_seq_len = current_seq_len;
    size_t new_total_len = old_seq_len + num_tokens;
    size_t elements_per_token = kv_heads * dim;
    size_t bytes_per_token = elements_per_token * element_size;
    size_t num_groups = (dim + KV_QUANT_GROUP_SIZE - 1) / KV_QUANT_GROUP_SIZE;
    size_t scales_per_token = kv_heads * num_groups;

    if (layer_idx == 0) {
        total_seq_len += num_tokens;
    }

    bool use_sliding_window = (window_size > 0 && new_total_len > window_size);
    size_t effective_seq_len = use_sliding_window ? window_size : new_total_len;

    if (!use_sliding_window) {
        size_t total_bytes = new_total_len * bytes_per_token;
        cache.keys.resize(total_bytes);
        cache.values.resize(total_bytes);

        size_t num_scales = new_total_len * scales_per_token;
        cache.key_scales.resize(num_scales);
        cache.value_scales.resize(num_scales);

        size_t dst_offset = old_seq_len * elements_per_token;
        size_t scale_offset = old_seq_len * scales_per_token;

        cactus_quantize_kv_fp16_to_int8(
            k_data,
            reinterpret_cast<int8_t*>(cache.keys.data()) + dst_offset,
            cache.key_scales.data() + scale_offset,
            num_tokens, kv_heads, dim);

        cactus_quantize_kv_fp16_to_int8(
            v_data,
            reinterpret_cast<int8_t*>(cache.values.data()) + dst_offset,
            cache.value_scales.data() + scale_offset,
            num_tokens, kv_heads, dim);
    } else {
        size_t cache_bytes = window_size * bytes_per_token;

        if (cache.keys.size() != cache_bytes) {
            cache.keys.resize(cache_bytes);
            cache.values.resize(cache_bytes);

            size_t num_scales = window_size * scales_per_token;
            cache.key_scales.resize(num_scales);
            cache.value_scales.resize(num_scales);
        }

        size_t remaining_window = window_size - sink_size;