| 74 | int64_t cache_steps, |
| 75 | int64_t step_elems, |
| 76 | std::vector<core::TensorValue> keys, |
| 77 | std::vector<core::TensorValue> values, |
| 78 | TransformerKVCacheOptions options) |
| 79 | : cache_steps_(std::max<int64_t>(0, cache_steps)), |
| 80 | step_elems_(std::max<int64_t>(0, step_elems)), |
| 81 | options_(options) { |
| 82 | if (step_elems_ <= 0) { |
| 83 | throw std::runtime_error("TransformerKVCache requires positive step_elems"); |
| 84 | } |
| 85 | if (keys.size() != values.size()) { |
| 86 | throw std::runtime_error("TransformerKVCache key/value layer counts must match"); |
| 87 | } |
| 88 | const size_t cache_elems = static_cast<size_t>(cache_steps_ * step_elems_); |
| 89 | layers_.reserve(keys.size()); |
| 90 | for (size_t layer = 0; layer < keys.size(); ++layer) { |
| 91 | validate_cache_tensor(keys[layer], options_); |
| 92 | validate_cache_tensor(values[layer], options_); |
| 93 | layers_.push_back(LayerCache{ |
| 94 | std::move(keys[layer]), |
| 95 | std::move(values[layer]), |
| 96 | std::vector<float>(cache_elems, 0.0F), |
nothing calls this directly
no test coverage detected