| 1212 | } |
| 1213 | |
| 1214 | common_init_result_ptr common_init_from_params(common_params & params) { |
| 1215 | common_init_result_ptr res(new common_init_result(params)); |
| 1216 | |
| 1217 | llama_model * model = res->model(); |
| 1218 | if (model == NULL) { |
| 1219 | LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); |
| 1220 | return res; |
| 1221 | } |
| 1222 | |
| 1223 | llama_context * lctx = res->context(); |
| 1224 | if (lctx == NULL) { |
| 1225 | LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); |
| 1226 | return res; |
| 1227 | } |
| 1228 | |
| 1229 | const llama_vocab * vocab = llama_model_get_vocab(model); |
| 1230 | |
| 1231 | if (params.ctx_shift && !llama_memory_can_shift(llama_get_memory(lctx))) { |
| 1232 | LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__); |
| 1233 | params.ctx_shift = false; |
| 1234 | } |
| 1235 | |
| 1236 | if (!params.control_vectors.empty()) { |
| 1237 | if (params.control_vector_layer_start <= 0) params.control_vector_layer_start = 1; |
| 1238 | if (params.control_vector_layer_end <= 0) params.control_vector_layer_end = llama_model_n_layer(model); |
| 1239 | |
| 1240 | const auto cvec = common_control_vector_load(params.control_vectors); |
| 1241 | if (cvec.n_embd == -1) { |
| 1242 | return res; |
| 1243 | } |
| 1244 | |
| 1245 | int err = llama_apply_adapter_cvec( |
| 1246 | lctx, |
| 1247 | cvec.data.data(), |
| 1248 | cvec.data.size(), |
| 1249 | cvec.n_embd, |
| 1250 | params.control_vector_layer_start, |
| 1251 | params.control_vector_layer_end); |
| 1252 | if (err) { |
| 1253 | return res; |
| 1254 | } |
| 1255 | } |
| 1256 | |
| 1257 | if (llama_pooling_type(lctx) == LLAMA_POOLING_TYPE_RANK) { |
| 1258 | bool ok = true; |
| 1259 | |
| 1260 | if (llama_vocab_bos(vocab) == LLAMA_TOKEN_NULL) { |
| 1261 | LOG_WRN("%s: warning: vocab does not have a BOS token, reranking will not work\n", __func__); |
| 1262 | ok = false; |
| 1263 | } |
| 1264 | |
| 1265 | bool has_eos = llama_vocab_eos(vocab) != LLAMA_TOKEN_NULL; |
| 1266 | bool has_sep = llama_vocab_sep(vocab) != LLAMA_TOKEN_NULL; |
| 1267 | bool has_rerank_prompt = llama_model_chat_template(model, "rerank") != NULL; |
| 1268 | |
| 1269 | if (!has_eos && !has_sep && !has_rerank_prompt) { |
| 1270 | LOG_WRN("%s: warning: vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n", __func__); |
| 1271 | ok = false; |