| 43 | int32_t get_num_physical_cores(); |
| 44 | |
| 45 | struct gpt_params { |
| 46 | uint32_t seed = -1; // RNG seed |
| 47 | |
| 48 | int32_t n_threads = get_num_physical_cores(); |
| 49 | int32_t n_threads_batch = -1; // number of threads to use for batch processing (-1 = use n_threads) |
| 50 | int32_t n_predict = -1; // new tokens to predict |
| 51 | int32_t n_ctx = 512; // context size |
| 52 | int32_t n_batch = 512; // batch size for prompt processing (must be >=32 to use BLAS) |
| 53 | int32_t n_keep = 0; // number of tokens to keep from initial prompt |
| 54 | int32_t n_draft = 16; // number of tokens to draft during speculative decoding |
| 55 | int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) |
| 56 | int32_t n_parallel = 1; // number of parallel sequences to decode |
| 57 | int32_t n_sequences = 1; // number of sequences to decode |
| 58 | float p_accept = 0.5f; // speculative decoding accept probability |
| 59 | float p_split = 0.1f; // speculative decoding split probability |
| 60 | int32_t n_gpu_layers = -1; // number of layers to store in VRAM (-1 - use default) |
| 61 | int32_t n_gpu_layers_draft = -1; // number of layers to store in VRAM for the draft model (-1 - use default) |
| 62 | int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors |
| 63 | float tensor_split[LLAMA_MAX_DEVICES] = {0}; // how split tensors should be distributed across GPUs |
| 64 | int32_t n_beams = 0; // if non-zero then use beam search of given width. |
| 65 | float rope_freq_base = 0.0f; // RoPE base frequency |
| 66 | float rope_freq_scale = 0.0f; // RoPE frequency scaling factor |
| 67 | float vram_budget_gb = -1.0f; // VRAM budget in GB (-1 - use available VRAM) |
| 68 | float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor |
| 69 | float yarn_attn_factor = 1.0f; // YaRN magnitude scaling factor |
| 70 | float yarn_beta_fast = 32.0f; // YaRN low correction dim |
| 71 | float yarn_beta_slow = 1.0f; // YaRN high correction dim |
| 72 | int32_t yarn_orig_ctx = 0; // YaRN original context length |
| 73 | int8_t rope_scaling_type = LLAMA_ROPE_SCALING_UNSPECIFIED; // TODO: better to be int32_t for alignment |
| 74 | // pinging @cebtenzzre |
| 75 | |
| 76 | // // sampling parameters |
| 77 | struct llama_sampling_params sparams; |
| 78 | |
| 79 | std::string model = "models/7B/ggml-model-f16.gguf"; // model path |
| 80 | std::string model_draft = ""; // draft model for speculative decoding |
| 81 | std::string model_alias = "unknown"; // model alias |
| 82 | std::string prompt = ""; |
| 83 | std::string prompt_file = ""; // store the external prompt file name |
| 84 | std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state |
| 85 | std::string input_prefix = ""; // string to prefix user inputs with |
| 86 | std::string input_suffix = ""; // string to suffix user inputs with |
| 87 | std::vector<std::string> antiprompt; // string upon seeing which more user input is prompted |
| 88 | std::string logdir = ""; // directory in which to save YAML log files |
| 89 | |
| 90 | // TODO: avoid tuple, use struct |
| 91 | std::vector<std::tuple<std::string, float>> lora_adapter; // lora adapter path with user defined scale |
| 92 | std::string lora_base = ""; // base model path for the lora adapter |
| 93 | |
| 94 | bool reset_gpu_index = false; // refresh the gpu index file |
| 95 | bool disale_gpu_index = false; // disable loading gpu index and splitting ffn |
| 96 | |
| 97 | int ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used. |
| 98 | int ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line |
| 99 | // (which is more convenient to use for plotting) |
| 100 | // |
| 101 | bool hellaswag = false; // compute HellaSwag score over random tasks from datafile supplied in prompt |
| 102 | size_t hellaswag_tasks = 400; // number of tasks to use when computing the HellaSwag score |
no test coverage detected