| 222 | server_task(server_task_type type) : type(type) {} |
| 223 | |
| 224 | static slot_params params_from_json_cmpl( |
| 225 | const llama_context * ctx, |
| 226 | const common_params & params_base, |
| 227 | const json & data) { |
| 228 | const llama_model * model = llama_get_model(ctx); |
| 229 | const llama_vocab * vocab = llama_model_get_vocab(model); |
| 230 | |
| 231 | slot_params params; |
| 232 | |
| 233 | // Sampling parameter defaults are loaded from the global server context (but individual requests can still override them) |
| 234 | slot_params defaults; |
| 235 | defaults.sampling = params_base.sampling; |
| 236 | defaults.speculative = params_base.speculative; |
| 237 | |
| 238 | // enabling this will output extra debug information in the HTTP responses from the server |
| 239 | params.verbose = params_base.verbosity > 9; |
| 240 | params.timings_per_token = json_value(data, "timings_per_token", false); |
| 241 | |
| 242 | params.stream = json_value(data, "stream", false); |
| 243 | params.cache_prompt = json_value(data, "cache_prompt", true); |
| 244 | params.return_tokens = json_value(data, "return_tokens", false); |
| 245 | params.n_predict = json_value(data, "n_predict", json_value(data, "max_tokens", defaults.n_predict)); |
| 246 | params.n_indent = json_value(data, "n_indent", defaults.n_indent); |
| 247 | params.n_keep = json_value(data, "n_keep", defaults.n_keep); |
| 248 | params.n_discard = json_value(data, "n_discard", defaults.n_discard); |
| 249 | //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement |
| 250 | params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); |
| 251 | params.response_fields = json_value(data, "response_fields", std::vector<std::string>()); |
| 252 | |
| 253 | params.sampling.top_k = json_value(data, "top_k", defaults.sampling.top_k); |
| 254 | params.sampling.top_p = json_value(data, "top_p", defaults.sampling.top_p); |
| 255 | params.sampling.min_p = json_value(data, "min_p", defaults.sampling.min_p); |
| 256 | params.sampling.top_n_sigma = json_value(data, "top_n_sigma", defaults.sampling.top_n_sigma); |
| 257 | params.sampling.xtc_probability = json_value(data, "xtc_probability", defaults.sampling.xtc_probability); |
| 258 | params.sampling.xtc_threshold = json_value(data, "xtc_threshold", defaults.sampling.xtc_threshold); |
| 259 | params.sampling.typ_p = json_value(data, "typical_p", defaults.sampling.typ_p); |
| 260 | params.sampling.temp = json_value(data, "temperature", defaults.sampling.temp); |
| 261 | params.sampling.dynatemp_range = json_value(data, "dynatemp_range", defaults.sampling.dynatemp_range); |
| 262 | params.sampling.dynatemp_exponent = json_value(data, "dynatemp_exponent", defaults.sampling.dynatemp_exponent); |
| 263 | params.sampling.penalty_last_n = json_value(data, "repeat_last_n", defaults.sampling.penalty_last_n); |
| 264 | params.sampling.penalty_repeat = json_value(data, "repeat_penalty", defaults.sampling.penalty_repeat); |
| 265 | params.sampling.penalty_freq = json_value(data, "frequency_penalty", defaults.sampling.penalty_freq); |
| 266 | params.sampling.penalty_present = json_value(data, "presence_penalty", defaults.sampling.penalty_present); |
| 267 | params.sampling.dry_multiplier = json_value(data, "dry_multiplier", defaults.sampling.dry_multiplier); |
| 268 | params.sampling.dry_base = json_value(data, "dry_base", defaults.sampling.dry_base); |
| 269 | params.sampling.dry_allowed_length = json_value(data, "dry_allowed_length", defaults.sampling.dry_allowed_length); |
| 270 | params.sampling.dry_penalty_last_n = json_value(data, "dry_penalty_last_n", defaults.sampling.dry_penalty_last_n); |
| 271 | params.sampling.mirostat = json_value(data, "mirostat", defaults.sampling.mirostat); |
| 272 | params.sampling.mirostat_tau = json_value(data, "mirostat_tau", defaults.sampling.mirostat_tau); |
| 273 | params.sampling.mirostat_eta = json_value(data, "mirostat_eta", defaults.sampling.mirostat_eta); |
| 274 | params.sampling.seed = json_value(data, "seed", defaults.sampling.seed); |
| 275 | params.sampling.n_probs = json_value(data, "n_probs", defaults.sampling.n_probs); |
| 276 | params.sampling.min_keep = json_value(data, "min_keep", defaults.sampling.min_keep); |
| 277 | params.post_sampling_probs = json_value(data, "post_sampling_probs", defaults.post_sampling_probs); |
| 278 | |
| 279 | params.speculative.n_min = json_value(data, "speculative.n_min", defaults.speculative.n_min); |
| 280 | params.speculative.n_max = json_value(data, "speculative.n_max", defaults.speculative.n_max); |
| 281 | params.speculative.p_min = json_value(data, "speculative.p_min", defaults.speculative.p_min); |
nothing calls this directly
no test coverage detected