MCPcopy Create free account
hub / github.com/BerriAI/litellm / evict_cache

Method evict_cache

litellm/caching/in_memory_cache.py:113–163  ·  view source on GitHub ↗

Eviction policy: 1. First, remove expired items from ttl_dict and cache_dict 2. If cache is still at or above max_size_in_memory, evict items with earliest expiration times This guarantees the following: - 1. When item ttl not set: At minimum each item will

(self)

Source from the content-addressed store, hash-verified

111 self.ttl_dict.pop(key, None)
112
113 def evict_cache(self):
114 """
115 Eviction policy:
116 1. First, remove expired items from ttl_dict and cache_dict
117 2. If cache is still at or above max_size_in_memory, evict items with earliest expiration times
118
119
120 This guarantees the following:
121 - 1. When item ttl not set: At minimum each item will remain in memory for the default ttl
122 - 2. When ttl is set: the item will remain in memory for at least that amount of time, unless cache size requires eviction
123 - 3. the size of in-memory cache is bounded
124
125 """
126 current_time: Final = self._clock()
127
128 # Step 1: Remove expired or outdated items
129 while self.expiration_heap:
130 expiration_time, key = self.expiration_heap[0]
131
132 # Case 1: Heap entry is outdated
133 if expiration_time != self.ttl_dict.get(key):
134 heapq.heappop(self.expiration_heap)
135 # Case 2: Entry is valid but expired
136 elif expiration_time <= current_time:
137 heapq.heappop(self.expiration_heap)
138 self._remove_key(key)
139 else:
140 # Case 3: Entry is valid and not expired
141 break
142
143 if len(self.expiration_heap) > 2 * self.max_size_in_memory:
144 self.expiration_heap = [entry for entry in self.expiration_heap if self.ttl_dict.get(entry[1]) == entry[0]]
145 heapq.heapify(self.expiration_heap)
146
147 # Step 2: Evict if cache is still full
148 while self.expiration_heap and len(self.cache_dict) >= self.max_size_in_memory:
149 expiration_time, key = heapq.heappop(self.expiration_heap)
150 # Skip if key was removed or updated
151 if self.ttl_dict.get(key) == expiration_time:
152 self._remove_key(key)
153
154 if not self.expiration_heap:
155 untracked_keys: Final = self._cache_keys()
156 overflow: Final = len(untracked_keys) - self.max_size_in_memory + 1
157 for untracked_key in untracked_keys[: max(overflow, 0)]:
158 self._remove_key(untracked_key)
159
160 # de-reference the removed item
161 # https://www.geeksforgeeks.org/diagnosing-and-fixing-memory-leaks-in-python/
162 # One of the most common causes of memory leaks in Python is the retention of objects that are no longer being used.
163 # This can occur when an object is referenced by another object, but the reference is never removed.
164
165 def allow_ttl_override(self, key: str) -> bool:
166 """

Callers 1

set_cacheMethod · 0.95

Calls 4

_remove_keyMethod · 0.95
_cache_keysMethod · 0.95
maxFunction · 0.85
getMethod · 0.65

Tested by

no test coverage detected