Blend base caches with anchor deltas weighted by similarity.
(
self,
ph_id: str,
message: str,
request_uid: str,
base_placeholder_cache: DynamicCache,
base_prefix_cache: DynamicCache,
anchor_list: List[Dict],
temperature: int = 1,
)
| 866 | } |
| 867 | |
| 868 | def offset_kv_cache_pair( |
| 869 | self, |
| 870 | ph_id: str, |
| 871 | message: str, |
| 872 | request_uid: str, |
| 873 | base_placeholder_cache: DynamicCache, |
| 874 | base_prefix_cache: DynamicCache, |
| 875 | anchor_list: List[Dict], |
| 876 | temperature: int = 1, |
| 877 | ) -> Tuple[DynamicCache, DynamicCache]: |
| 878 | """Blend base caches with anchor deltas weighted by similarity.""" |
| 879 | placeholder_len = int(base_placeholder_cache._seen_tokens) |
| 880 | if placeholder_len <= 0: |
| 881 | self._log_warning("real_placeholder_kv_cache has no tokens, skip updating.") |
| 882 | return base_placeholder_cache, base_prefix_cache |
| 883 | |
| 884 | real_key_embedding, real_value_embedding = self._stack_cache_tensors(base_placeholder_cache) |
| 885 | |
| 886 | anchor_signature = self.anchor_signature(anchor_list) |
| 887 | |
| 888 | cache_entry = self._get_cached_anchor_weights( |
| 889 | request_uid, |
| 890 | ph_id, |
| 891 | message=message, |
| 892 | signature=anchor_signature, |
| 893 | ) |
| 894 | |
| 895 | if cache_entry is None: |
| 896 | anchor_index = self._select_anchor_indices(anchor_list, placeholder_len) |
| 897 | if not anchor_index: |
| 898 | if anchor_list: |
| 899 | self._log_warning( |
| 900 | f"No anchors cover placeholder {ph_id} for Agent {self.llm.node_id} ({self.llm.role})." |
| 901 | ) |
| 902 | return base_placeholder_cache.copy(), base_prefix_cache.copy() |
| 903 | |
| 904 | cache_entry = self._compute_anchor_weight_entry( |
| 905 | anchor_list, |
| 906 | anchor_index, |
| 907 | real_key_embedding, |
| 908 | real_value_embedding, |
| 909 | placeholder_len, |
| 910 | float(temperature), |
| 911 | ) |
| 912 | if cache_entry is None: |
| 913 | return base_placeholder_cache.copy(), base_prefix_cache.copy() |
| 914 | cache_entry["anchor_signature"] = anchor_signature |
| 915 | cache_entry["placeholder_len"] = placeholder_len |
| 916 | self._set_cached_anchor_weights(request_uid, ph_id, message, cache_entry) |
| 917 | else: |
| 918 | anchor_index = cache_entry["anchor_index"] |
| 919 | if not anchor_index: |
| 920 | return base_placeholder_cache.copy(), base_prefix_cache.copy() |
| 921 | |
| 922 | weights_key_for_prefix = cache_entry["weights_key_for_prefix"] |
| 923 | weights_value_for_prefix = cache_entry["weights_value_for_prefix"] |
| 924 | weights_key_for_placeholder = cache_entry["weights_key_for_placeholder"] |
| 925 | weights_value_for_placeholder = cache_entry["weights_value_for_placeholder"] |
no test coverage detected