Class LlamaContext

llama_cpp/_internals.py:247–464

Intermediate Python wrapper for a llama.cpp llama_context. NOTE: For stability it's recommended you use the Llama class instead.

class LlamaContext:
    """Intermediate Python wrapper for a llama.cpp llama_context.
    NOTE: For stability it's recommended you use the Llama class instead."""

    def __init__(
        self,
        *,
        model: LlamaModel,
        params: llama_cpp.llama_context_params,
        verbose: bool = True,
    ):
        self.model = model
        self.params = params
        self.verbose = verbose
        self._exit_stack = ExitStack()

        ctx = llama_cpp.llama_init_from_model(self.model.model, self.params)

        if ctx is None:
            raise ValueError("Failed to create llama_context")

        self.ctx = ctx
        self.memory = llama_cpp.llama_get_memory(self.ctx)
        self.sampler = None  # LlamaContext doesn't manage samplers directly, but some cleanup code expects this attribute

        def free_ctx():
            if self.ctx is None:
                return
            llama_cpp.llama_free(self.ctx)
            self.ctx = None

        self._exit_stack.callback(free_ctx)

    def close(self):
        self._exit_stack.close()

    def __del__(self):
        self.close()

    def n_ctx(self) -> int:
        return llama_cpp.llama_n_ctx(self.ctx)

    def pooling_type(self) -> int:
        return llama_cpp.llama_pooling_type(self.ctx)

    def kv_cache_clear(self):
        # Embedding models with non-causal attention may not allocate memory.
        if self.memory is None:
            return
        llama_cpp.llama_memory_clear(self.memory, True)

    def kv_cache_seq_rm(self, seq_id: int, p0: int, p1: int) -> bool:
        assert self.memory is not None, "Memory is not initialized"
        seq_id = seq_id if seq_id >= 0 else 0
        return llama_cpp.llama_memory_seq_rm(self.memory, seq_id, p0, p1)

    def kv_cache_seq_cp(self, seq_id_src: int, seq_id_dst: int, p0: int, p1: int):
        assert self.memory is not None, "Memory is not initialized"