(
self,
*,
model_path: str,
model_alias: Optional[str] = None,
chat_template: Optional[str] = None,
loras: Optional[List["Model.LoraAdapter"]] = None,
n_gpu_layers: Optional[int] = None,
split_mode: Optional[int] = None,
main_gpu: Optional[int] = None,
tensor_split: Optional[List[float]] = None,
vocab_only: Optional[bool] = None,
use_mmap: Optional[bool] = None,
use_mlock: Optional[bool] = None,
kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None,
n_ctx: Optional[int],
n_batch: Optional[int],
n_ubatch: Optional[int] = None,
n_seq_max: Optional[int],
n_threads: Optional[int],
n_threads_batch: Optional[int],
rope_scaling_type: Optional[int] = None,
pooling_type: Optional[int] = None,
attention_type: Optional[int] = None,
embedding: Optional[bool] = None,
rope_freq_base: Optional[float] = None,
rope_freq_scale: Optional[float] = None,
yarn_ext_factor: Optional[float] = None,
yarn_attn_factor: Optional[float] = None,
yarn_beta_fast: Optional[float] = None,
yarn_beta_slow: Optional[float] = None,
yarn_orig_ctx: Optional[int] = None,
offload_kqv: Optional[bool] = None,
flash_attn: Optional[bool] = None,
op_offload: Optional[bool] = None,
swa_full: Optional[bool] = None,
no_perf: Optional[bool] = None,
type_k: Optional[int] = None,
type_v: Optional[int] = None,
kv_unified: bool = True,
max_seq_len: Optional[int] = None,
max_output_tokens: Optional[int] = None,
draft_model: Optional[str] = None,
draft_model_path: Optional[str] = None,
draft_model_num_pred_tokens: int = 16,
draft_model_max_ngram_size: int = 2,
draft_model_top_k: int = 1,
draft_model_p_min: float = 0.0,
draft_model_max_batch_size: Optional[int] = None,
draft_model_threads: Optional[int] = None,
draft_model_threads_batch: Optional[int] = None,
response_schema: Optional[Dict[str, Any]] = None,
store_logits: bool = True,
)
| 11233 | scale: float = 1.0 |
| 11234 | |
| 11235 | def __init__( |
| 11236 | self, |
| 11237 | *, |
| 11238 | model_path: str, |
| 11239 | model_alias: Optional[str] = None, |
| 11240 | chat_template: Optional[str] = None, |
| 11241 | loras: Optional[List["Model.LoraAdapter"]] = None, |
| 11242 | n_gpu_layers: Optional[int] = None, |
| 11243 | split_mode: Optional[int] = None, |
| 11244 | main_gpu: Optional[int] = None, |
| 11245 | tensor_split: Optional[List[float]] = None, |
| 11246 | vocab_only: Optional[bool] = None, |
| 11247 | use_mmap: Optional[bool] = None, |
| 11248 | use_mlock: Optional[bool] = None, |
| 11249 | kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]] = None, |
| 11250 | n_ctx: Optional[int], |
| 11251 | n_batch: Optional[int], |
| 11252 | n_ubatch: Optional[int] = None, |
| 11253 | n_seq_max: Optional[int], |
| 11254 | n_threads: Optional[int], |
| 11255 | n_threads_batch: Optional[int], |
| 11256 | rope_scaling_type: Optional[int] = None, |
| 11257 | pooling_type: Optional[int] = None, |
| 11258 | attention_type: Optional[int] = None, |
| 11259 | embedding: Optional[bool] = None, |
| 11260 | rope_freq_base: Optional[float] = None, |
| 11261 | rope_freq_scale: Optional[float] = None, |
| 11262 | yarn_ext_factor: Optional[float] = None, |
| 11263 | yarn_attn_factor: Optional[float] = None, |
| 11264 | yarn_beta_fast: Optional[float] = None, |
| 11265 | yarn_beta_slow: Optional[float] = None, |
| 11266 | yarn_orig_ctx: Optional[int] = None, |
| 11267 | offload_kqv: Optional[bool] = None, |
| 11268 | flash_attn: Optional[bool] = None, |
| 11269 | op_offload: Optional[bool] = None, |
| 11270 | swa_full: Optional[bool] = None, |
| 11271 | no_perf: Optional[bool] = None, |
| 11272 | type_k: Optional[int] = None, |
| 11273 | type_v: Optional[int] = None, |
| 11274 | kv_unified: bool = True, |
| 11275 | max_seq_len: Optional[int] = None, |
| 11276 | max_output_tokens: Optional[int] = None, |
| 11277 | draft_model: Optional[str] = None, |
| 11278 | draft_model_path: Optional[str] = None, |
| 11279 | draft_model_num_pred_tokens: int = 16, |
| 11280 | draft_model_max_ngram_size: int = 2, |
| 11281 | draft_model_top_k: int = 1, |
| 11282 | draft_model_p_min: float = 0.0, |
| 11283 | draft_model_max_batch_size: Optional[int] = None, |
| 11284 | draft_model_threads: Optional[int] = None, |
| 11285 | draft_model_threads_batch: Optional[int] = None, |
| 11286 | response_schema: Optional[Dict[str, Any]] = None, |
| 11287 | store_logits: bool = True, |
| 11288 | ) -> None: |
| 11289 | llama_cpp.llama_backend_init() |
| 11290 | self.backend_initialized = True |
| 11291 | self.model_path = model_path |
| 11292 | self.model_alias = model_alias |
nothing calls this directly
no test coverage detected