| 11580 | |
| 11581 | @staticmethod |
| 11582 | def build_model_params( |
| 11583 | *, |
| 11584 | n_gpu_layers: Optional[int], |
| 11585 | split_mode: Optional[int], |
| 11586 | main_gpu: Optional[int], |
| 11587 | tensor_split: Optional[List[float]], |
| 11588 | vocab_only: Optional[bool], |
| 11589 | use_mmap: Optional[bool], |
| 11590 | use_mlock: Optional[bool], |
| 11591 | kv_overrides: Optional[Dict[str, Union[bool, int, float, str]]], |
| 11592 | ) -> Tuple[Any, Optional[Any], Optional[Any]]: |
| 11593 | model_params = llama_cpp.llama_model_default_params() |
| 11594 | if n_gpu_layers is not None: |
| 11595 | model_params.n_gpu_layers = 0x7FFFFFFF if n_gpu_layers == -1 else n_gpu_layers |
| 11596 | if split_mode is not None: |
| 11597 | model_params.split_mode = split_mode |
| 11598 | if main_gpu is not None: |
| 11599 | model_params.main_gpu = main_gpu |
| 11600 | tensor_split_ref = None |
| 11601 | if tensor_split is not None: |
| 11602 | if len(tensor_split) > llama_cpp.LLAMA_MAX_DEVICES: |
| 11603 | raise ValueError( |
| 11604 | "tensor_split exceeds maximum supported devices " |
| 11605 | f"({llama_cpp.LLAMA_MAX_DEVICES})" |
| 11606 | ) |
| 11607 | float_array = ctypes.c_float * llama_cpp.LLAMA_MAX_DEVICES |
| 11608 | tensor_split_ref = float_array(*tensor_split) |
| 11609 | model_params.tensor_split = tensor_split_ref |
| 11610 | if vocab_only is not None: |
| 11611 | model_params.vocab_only = vocab_only |
| 11612 | if use_mmap is not None: |
| 11613 | model_params.use_mmap = use_mmap |
| 11614 | if use_mlock is not None: |
| 11615 | model_params.use_mlock = use_mlock |
| 11616 | |
| 11617 | kv_overrides_ref = None |
| 11618 | if kv_overrides is not None: |
| 11619 | kv_overrides_ref = ( |
| 11620 | llama_cpp.llama_model_kv_override * (len(kv_overrides) + 1) |
| 11621 | )() |
| 11622 | for index, (key, value) in enumerate(kv_overrides.items()): |
| 11623 | kv_overrides_ref[index].key = key.encode("utf-8") |
| 11624 | if isinstance(value, bool): |
| 11625 | kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_BOOL |
| 11626 | kv_overrides_ref[index].value.val_bool = value |
| 11627 | elif isinstance(value, int): |
| 11628 | kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_INT |
| 11629 | kv_overrides_ref[index].value.val_i64 = value |
| 11630 | elif isinstance(value, float): |
| 11631 | kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_FLOAT |
| 11632 | kv_overrides_ref[index].value.val_f64 = value |
| 11633 | elif isinstance(value, str): |
| 11634 | encoded = value.encode("utf-8") |
| 11635 | if len(encoded) > 128: |
| 11636 | raise ValueError(f"kv_overrides value for {key} is too long") |
| 11637 | encoded = encoded.ljust(128, b"\0") |
| 11638 | kv_overrides_ref[index].tag = llama_cpp.LLAMA_KV_OVERRIDE_TYPE_STR |
| 11639 | address = cast( |