(self, capacity_bytes: int = (2 << 30))
| 50 | """Cache for a llama.cpp model using RAM.""" |
| 51 | |
| 52 | def __init__(self, capacity_bytes: int = (2 << 30)): |
| 53 | super().__init__(capacity_bytes) |
| 54 | self.capacity_bytes = capacity_bytes |
| 55 | self.cache_state: OrderedDict[Tuple[int, ...], "llama_cpp.llama.LlamaState"] = ( |
| 56 | OrderedDict() |
| 57 | ) |
| 58 | |
| 59 | @property |
| 60 | def cache_size(self): |