| 9 | |
| 10 | namespace engine::runtime { |
| 11 | |
| 12 | namespace { |
| 13 | |
| 14 | void validate_cache_tensor(const core::TensorValue & tensor, const TransformerKVCacheOptions & options) { |
| 15 | if (tensor.type == GGML_TYPE_F32) { |
| 16 | return; |
| 17 | } |
| 18 | if (options.allow_f16_storage && tensor.type == GGML_TYPE_F16) { |
| 19 | return; |
| 20 | } |
| 21 | if (options.allow_bf16_storage && tensor.type == GGML_TYPE_BF16) { |
| 22 | return; |
| 23 | } |
| 24 | throw std::runtime_error( |
| 25 | options.allow_f16_storage || options.allow_bf16_storage |
| 26 | ? "TransformerKVCache supports only f32/f16/bf16 cache tensors when enabled" |
| 27 | : "TransformerKVCache requires f32 cache tensors"); |
| 28 | } |
| 29 | |
| 30 | void write_cache_tensor( |
| 31 | const core::TensorValue & tensor, |
| 32 | const std::vector<float> & values, |
| 33 | const TransformerKVCacheOptions & options) { |
| 34 | validate_cache_tensor(tensor, options); |
| 35 | if (tensor.type == GGML_TYPE_F32) { |
| 36 | core::write_tensor_f32(tensor, values); |
| 37 | return; |