Load a llama.cpp model from `model_path`. Examples: Basic usage >>> import llama_cpp >>> model = llama_cpp.Llama( ... model_path="path/to/model", ... ) >>> print(model("The quick brown fox jumps ", stop=["."])["cho
(
self,
model_path: str,
*,
# Model Params
n_gpu_layers: int = 0,
split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER,
main_gpu: int = 0,
tensor_split: Optional[List[float]] = None,
vocab_only: bool = False,
use_mmap: bool = True,
use_mlock: bool = False,
kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None,
# Context Params
seed: int = llama_cpp.LLAMA_DEFAULT_SEED,
n_ctx: int = 512,
n_batch: int = 512,
n_ubatch: int = 512,
n_threads: Optional[int] = None,
n_threads_batch: Optional[int] = None,
rope_scaling_type: Optional[
int
] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED,
pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED,
attention_type: int = llama_cpp.LLAMA_ATTENTION_TYPE_UNSPECIFIED,
rope_freq_base: float = 0.0,
rope_freq_scale: float = 0.0,
yarn_ext_factor: float = -1.0,
yarn_attn_factor: float = 1.0,
yarn_beta_fast: float = 32.0,
yarn_beta_slow: float = 1.0,
yarn_orig_ctx: int = 0,
logits_all: bool = False,
embedding: bool = False,
offload_kqv: bool = True,
flash_attn: bool = False,
op_offload: Optional[bool] = None,
swa_full: Optional[bool] = None,
# Sampling Params
no_perf: bool = False,
last_n_tokens_size: int = 64,
# LoRA Params
lora_base: Optional[str] = None,
lora_scale: float = 1.0,
lora_path: Optional[str] = None,
# Backend Params
numa: Union[bool, int] = False,
# Chat Format Params
chat_format: Optional[str] = None,
chat_handler: Optional[llama_chat_format.LlamaChatCompletionHandler] = None,
# Speculative Decoding
draft_model: Optional[LlamaDraftModel] = None,
# Tokenizer Override
tokenizer: Optional[BaseLlamaTokenizer] = None,
# KV cache quantization
type_k: Optional[int] = None,
type_v: Optional[int] = None,
# Misc
spm_infill: bool = False,
verbose: bool = True,
# Extra Params
**kwargs, # type: ignore
)
| 58 | __backend_initialized = False |
| 59 | |
| 60 | def __init__( |
| 61 | self, |
| 62 | model_path: str, |
| 63 | *, |
| 64 | # Model Params |
| 65 | n_gpu_layers: int = 0, |
| 66 | split_mode: int = llama_cpp.LLAMA_SPLIT_MODE_LAYER, |
| 67 | main_gpu: int = 0, |
| 68 | tensor_split: Optional[List[float]] = None, |
| 69 | vocab_only: bool = False, |
| 70 | use_mmap: bool = True, |
| 71 | use_mlock: bool = False, |
| 72 | kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, |
| 73 | # Context Params |
| 74 | seed: int = llama_cpp.LLAMA_DEFAULT_SEED, |
| 75 | n_ctx: int = 512, |
| 76 | n_batch: int = 512, |
| 77 | n_ubatch: int = 512, |
| 78 | n_threads: Optional[int] = None, |
| 79 | n_threads_batch: Optional[int] = None, |
| 80 | rope_scaling_type: Optional[ |
| 81 | int |
| 82 | ] = llama_cpp.LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED, |
| 83 | pooling_type: int = llama_cpp.LLAMA_POOLING_TYPE_UNSPECIFIED, |
| 84 | attention_type: int = llama_cpp.LLAMA_ATTENTION_TYPE_UNSPECIFIED, |
| 85 | rope_freq_base: float = 0.0, |
| 86 | rope_freq_scale: float = 0.0, |
| 87 | yarn_ext_factor: float = -1.0, |
| 88 | yarn_attn_factor: float = 1.0, |
| 89 | yarn_beta_fast: float = 32.0, |
| 90 | yarn_beta_slow: float = 1.0, |
| 91 | yarn_orig_ctx: int = 0, |
| 92 | logits_all: bool = False, |
| 93 | embedding: bool = False, |
| 94 | offload_kqv: bool = True, |
| 95 | flash_attn: bool = False, |
| 96 | op_offload: Optional[bool] = None, |
| 97 | swa_full: Optional[bool] = None, |
| 98 | # Sampling Params |
| 99 | no_perf: bool = False, |
| 100 | last_n_tokens_size: int = 64, |
| 101 | # LoRA Params |
| 102 | lora_base: Optional[str] = None, |
| 103 | lora_scale: float = 1.0, |
| 104 | lora_path: Optional[str] = None, |
| 105 | # Backend Params |
| 106 | numa: Union[bool, int] = False, |
| 107 | # Chat Format Params |
| 108 | chat_format: Optional[str] = None, |
| 109 | chat_handler: Optional[llama_chat_format.LlamaChatCompletionHandler] = None, |
| 110 | # Speculative Decoding |
| 111 | draft_model: Optional[LlamaDraftModel] = None, |
| 112 | # Tokenizer Override |
| 113 | tokenizer: Optional[BaseLlamaTokenizer] = None, |
| 114 | # KV cache quantization |
| 115 | type_k: Optional[int] = None, |
| 116 | type_v: Optional[int] = None, |
| 117 | # Misc |
no test coverage detected