Extract a cache entry for exclusive use. If the entry has count > 1, deep copy and decrement. If count == 1, remove from cache entirely.
(self, model, tokens: Tuple[int, ...])
| 148 | del d_prev[t] |
| 149 | |
| 150 | def _extract(self, model, tokens: Tuple[int, ...]) -> CacheEntry: |
| 151 | """ |
| 152 | Extract a cache entry for exclusive use. |
| 153 | |
| 154 | If the entry has count > 1, deep copy and decrement. |
| 155 | If count == 1, remove from cache entirely. |
| 156 | """ |
| 157 | cache_entry = self._get(model, tokens) |
| 158 | if cache_entry.count == 1: |
| 159 | self._delete(model, tokens) |
| 160 | self._lru.remove((model, tokens)) |
| 161 | return cache_entry |
| 162 | |
| 163 | cache_entry.count -= 1 |
| 164 | return CacheEntry( |
| 165 | copy.deepcopy(cache_entry.prompt_cache), |
| 166 | 1, |
| 167 | ) |
| 168 | |
| 169 | def fetch_nearest_cache( |
| 170 | self, model, tokens: List[int] |
no test coverage detected