| 1908 | } |
| 1909 | |
| 1910 | bool load_model(const common_params & params) { |
| 1911 | SRV_INF("loading model '%s'\n", params.model.path.c_str()); |
| 1912 | |
| 1913 | params_base = params; |
| 1914 | |
| 1915 | llama_init = common_init_from_params(params_base); |
| 1916 | |
| 1917 | model = llama_init.model.get(); |
| 1918 | ctx = llama_init.context.get(); |
| 1919 | |
| 1920 | if (model == nullptr) { |
| 1921 | SRV_ERR("failed to load model, '%s'\n", params_base.model.path.c_str()); |
| 1922 | return false; |
| 1923 | } |
| 1924 | |
| 1925 | vocab = llama_model_get_vocab(model); |
| 1926 | |
| 1927 | n_ctx = llama_n_ctx(ctx); |
| 1928 | |
| 1929 | add_bos_token = llama_vocab_get_add_bos(vocab); |
| 1930 | has_eos_token = llama_vocab_eos(vocab) != LLAMA_TOKEN_NULL; |
| 1931 | |
| 1932 | if (!params_base.speculative.model.path.empty() || !params_base.speculative.model.hf_repo.empty()) { |
| 1933 | SRV_INF("loading draft model '%s'\n", params_base.speculative.model.path.c_str()); |
| 1934 | |
| 1935 | auto params_dft = params_base; |
| 1936 | |
| 1937 | params_dft.devices = params_base.speculative.devices; |
| 1938 | params_dft.model = params_base.speculative.model; |
| 1939 | params_dft.n_ctx = params_base.speculative.n_ctx == 0 ? params_base.n_ctx / params_base.n_parallel : params_base.speculative.n_ctx; |
| 1940 | params_dft.n_gpu_layers = params_base.speculative.n_gpu_layers; |
| 1941 | params_dft.n_parallel = 1; |
| 1942 | |
| 1943 | // force F16 KV cache for the draft model for extra performance |
| 1944 | params_dft.cache_type_k = GGML_TYPE_F16; |
| 1945 | params_dft.cache_type_v = GGML_TYPE_F16; |
| 1946 | |
| 1947 | llama_init_dft = common_init_from_params(params_dft); |
| 1948 | |
| 1949 | model_dft = llama_init_dft.model.get(); |
| 1950 | |
| 1951 | if (model_dft == nullptr) { |
| 1952 | SRV_ERR("failed to load draft model, '%s'\n", params_base.speculative.model.path.c_str()); |
| 1953 | return false; |
| 1954 | } |
| 1955 | |
| 1956 | if (!common_speculative_are_compatible(ctx, llama_init_dft.context.get())) { |
| 1957 | SRV_ERR("the draft model '%s' is not compatible with the target model '%s'\n", params_base.speculative.model.path.c_str(), params_base.model.path.c_str()); |
| 1958 | |
| 1959 | return false; |
| 1960 | } |
| 1961 | |
| 1962 | const int n_ctx_dft = llama_n_ctx(llama_init_dft.context.get()); |
| 1963 | |
| 1964 | cparams_dft = common_context_params_to_llama(params_dft); |
| 1965 | cparams_dft.n_batch = n_ctx_dft; |
| 1966 | |
| 1967 | // the context is not needed - we will create one for each slot |
no test coverage detected