| 67 | } |
| 68 | |
| 69 | int main(int argc, char ** argv) { |
| 70 | // own arguments required by this example |
| 71 | common_params params; |
| 72 | |
| 73 | if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_SERVER)) { |
| 74 | return 1; |
| 75 | } |
| 76 | |
| 77 | // validate batch size for embeddings |
| 78 | // embeddings require all tokens to be processed in a single ubatch |
| 79 | // see https://github.com/ggml-org/llama.cpp/issues/12836 |
| 80 | if (params.embedding && params.n_batch > params.n_ubatch) { |
| 81 | LOG_WRN("%s: embeddings enabled with n_batch (%d) > n_ubatch (%d)\n", __func__, params.n_batch, params.n_ubatch); |
| 82 | LOG_WRN("%s: setting n_batch = n_ubatch = %d to avoid assertion failure\n", __func__, params.n_ubatch); |
| 83 | params.n_batch = params.n_ubatch; |
| 84 | } |
| 85 | |
| 86 | if (params.n_parallel < 0) { |
| 87 | LOG_INF("%s: n_parallel is set to auto, using n_parallel = 4 and kv_unified = true\n", __func__); |
| 88 | |
| 89 | params.n_parallel = 4; |
| 90 | params.kv_unified = true; |
| 91 | } |
| 92 | |
| 93 | // for consistency between server router mode and single-model mode, we set the same model name as alias |
| 94 | if (params.model_alias.empty() && !params.model.name.empty()) { |
| 95 | params.model_alias = params.model.name; |
| 96 | } |
| 97 | |
| 98 | common_init(); |
| 99 | |
| 100 | // struct that contains llama context and inference |
| 101 | server_context ctx_server; |
| 102 | |
| 103 | llama_backend_init(); |
| 104 | llama_numa_init(params.numa); |
| 105 | |
| 106 | LOG_INF("system info: n_threads = %d, n_threads_batch = %d, total_threads = %d\n", params.cpuparams.n_threads, params.cpuparams_batch.n_threads, std::thread::hardware_concurrency()); |
| 107 | LOG_INF("\n"); |
| 108 | LOG_INF("%s\n", common_params_get_system_info(params).c_str()); |
| 109 | LOG_INF("\n"); |
| 110 | |
| 111 | server_http_context ctx_http; |
| 112 | if (!ctx_http.init(params)) { |
| 113 | LOG_ERR("%s: failed to initialize HTTP server\n", __func__); |
| 114 | return 1; |
| 115 | } |
| 116 | |
| 117 | // |
| 118 | // Router |
| 119 | // |
| 120 | |
| 121 | // register API routes |
| 122 | server_routes routes(params, ctx_server); |
| 123 | |
| 124 | bool is_router_server = params.model.path.empty(); |
| 125 | std::optional<server_models_routes> models_routes{}; |
| 126 | if (is_router_server) { |
nothing calls this directly
no test coverage detected