| 238 | // |
| 239 | |
| 240 | task_params server_task::params_from_json_cmpl( |
| 241 | const llama_vocab * vocab, |
| 242 | const common_params & params_base, |
| 243 | const int n_ctx_slot, |
| 244 | const std::vector<llama_logit_bias> & logit_bias_eog, |
| 245 | const json & data) { |
| 246 | task_params params; |
| 247 | |
| 248 | // Sampling parameter defaults are loaded from the global server context (but individual requests can still them) |
| 249 | task_params defaults; |
| 250 | defaults.sampling = params_base.sampling; |
| 251 | defaults.speculative = params_base.speculative; |
| 252 | defaults.n_keep = params_base.n_keep; |
| 253 | defaults.n_predict = params_base.n_predict; |
| 254 | defaults.n_cache_reuse = params_base.n_cache_reuse; |
| 255 | defaults.cache_prompt = params_base.cache_prompt; |
| 256 | defaults.antiprompt = params_base.antiprompt; |
| 257 | |
| 258 | // enabling this will output extra debug information in the HTTP responses from the server |
| 259 | params.verbose = params_base.verbosity > 9; |
| 260 | params.timings_per_token = json_value(data, "timings_per_token", false); |
| 261 | |
| 262 | params.stream = json_value(data, "stream", false); |
| 263 | auto stream_opt = json_value(data, "stream_options", json::object()); |
| 264 | params.include_usage = json_value(stream_opt, "include_usage", false); |
| 265 | params.cache_prompt = json_value(data, "cache_prompt", defaults.cache_prompt); |
| 266 | params.return_tokens = json_value(data, "return_tokens", false); |
| 267 | params.return_progress = json_value(data, "return_progress", false); |
| 268 | auto max_tokens = json_value(data, "max_tokens", defaults.n_predict); |
| 269 | params.n_predict = json_value(data, "n_predict", json_value(data, "max_completion_tokens", max_tokens)); |
| 270 | params.n_indent = json_value(data, "n_indent", defaults.n_indent); |
| 271 | params.n_keep = json_value(data, "n_keep", defaults.n_keep); |
| 272 | params.n_discard = json_value(data, "n_discard", defaults.n_discard); |
| 273 | params.n_discard = std::max(0, params.n_discard); |
| 274 | params.n_cmpl = json_value(data, "n_cmpl", json_value(data, "n", 1)); |
| 275 | params.n_cache_reuse = json_value(data, "n_cache_reuse", defaults.n_cache_reuse); |
| 276 | //params.t_max_prompt_ms = json_value(data, "t_max_prompt_ms", defaults.t_max_prompt_ms); // TODO: implement |
| 277 | params.t_max_predict_ms = json_value(data, "t_max_predict_ms", defaults.t_max_predict_ms); |
| 278 | params.response_fields = json_value(data, "response_fields", std::vector<std::string>()); |
| 279 | |
| 280 | params.sampling.top_k = json_value(data, "top_k", defaults.sampling.top_k); |
| 281 | params.sampling.top_p = json_value(data, "top_p", defaults.sampling.top_p); |
| 282 | params.sampling.min_p = json_value(data, "min_p", defaults.sampling.min_p); |
| 283 | params.sampling.top_n_sigma = json_value(data, "top_n_sigma", defaults.sampling.top_n_sigma); |
| 284 | params.sampling.xtc_probability = json_value(data, "xtc_probability", defaults.sampling.xtc_probability); |
| 285 | params.sampling.xtc_threshold = json_value(data, "xtc_threshold", defaults.sampling.xtc_threshold); |
| 286 | params.sampling.typ_p = json_value(data, "typical_p", defaults.sampling.typ_p); |
| 287 | params.sampling.temp = json_value(data, "temperature", defaults.sampling.temp); |
| 288 | params.sampling.dynatemp_range = json_value(data, "dynatemp_range", defaults.sampling.dynatemp_range); |
| 289 | params.sampling.dynatemp_exponent = json_value(data, "dynatemp_exponent", defaults.sampling.dynatemp_exponent); |
| 290 | params.sampling.penalty_last_n = json_value(data, "repeat_last_n", defaults.sampling.penalty_last_n); |
| 291 | params.sampling.penalty_repeat = json_value(data, "repeat_penalty", defaults.sampling.penalty_repeat); |
| 292 | params.sampling.penalty_freq = json_value(data, "frequency_penalty", defaults.sampling.penalty_freq); |
| 293 | params.sampling.penalty_present = json_value(data, "presence_penalty", defaults.sampling.penalty_present); |
| 294 | params.sampling.dry_multiplier = json_value(data, "dry_multiplier", defaults.sampling.dry_multiplier); |
| 295 | params.sampling.dry_base = json_value(data, "dry_base", defaults.sampling.dry_base); |
| 296 | params.sampling.dry_allowed_length = json_value(data, "dry_allowed_length", defaults.sampling.dry_allowed_length); |
| 297 | params.sampling.dry_penalty_last_n = json_value(data, "dry_penalty_last_n", defaults.sampling.dry_penalty_last_n); |
nothing calls this directly
no test coverage detected